349 lines
13 KiB
Python
349 lines
13 KiB
Python
|
import requests
|
||
|
|
import json
|
||
|
|
import config
|
||
|
|
import dpkt
|
||
|
|
import datetime
|
||
|
|
import pprint
|
||
|
|
|
||
|
|
from bitstring import BitStream
|
||
|
|
from flask import render_template, request, make_response, redirect
|
||
|
|
|
||
|
|
from . import app
|
||
|
|
from .search_form import SearchForm
|
||
|
|
from .add_service_form import AddServiceForm
|
||
|
|
from .add_host_form import AddHostForm
|
||
|
|
from .data import Search, load_metadata, local_pcap_path
|
||
|
|
|
||
|
|
def cutoff(string, maximum):
|
||
|
|
if len(string) > maximum - 3:
|
||
|
|
string = string[:maximum] + '...'
|
||
|
|
return string
|
||
|
|
|
||
|
|
app.jinja_env.globals.update(tz_offset=config.TIMEZONE_OFFSET)
|
||
|
|
app.jinja_env.globals.update(load_metadata=load_metadata)
|
||
|
|
app.jinja_env.globals.update(pprint=pprint.pformat)
|
||
|
|
app.jinja_env.globals.update(cutoff=cutoff)
|
||
|
|
app.jinja_env.globals.update(json=json)
|
||
|
|
|
||
|
|
@app.route('/')
|
||
|
|
@app.route('/history')
|
||
|
|
def show_history():
|
||
|
|
all_searches = sorted(Search.loadall(partial=True), key=lambda x: x.num, reverse=True)
|
||
|
|
return render_template('search_history.html', searches=all_searches, title='Search History')
|
||
|
|
|
||
|
|
def to_boolean(value):
|
||
|
|
if value == '1':
|
||
|
|
return True
|
||
|
|
elif value == '2':
|
||
|
|
return False
|
||
|
|
else:
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def jsonify_params(params, bool_params=None, int_params=None, float_params=None, string_params=None,
|
||
|
|
list_params=None, timestamp_params=None, composite_params=None, custom_params=None):
|
||
|
|
params_json_dict = {}
|
||
|
|
all_custom_key_fields = list()
|
||
|
|
if custom_params:
|
||
|
|
for custom_key in custom_params:
|
||
|
|
all_custom_key_fields.extend(custom_params[custom_key])
|
||
|
|
|
||
|
|
# Sanitize and convert parameters
|
||
|
|
for key, value in params.iteritems():
|
||
|
|
if str(key)[-1].isdigit():
|
||
|
|
base_key = str(key)[:-2]
|
||
|
|
else:
|
||
|
|
base_key = key
|
||
|
|
|
||
|
|
value = params[key]
|
||
|
|
if value == '':
|
||
|
|
params[key] = None
|
||
|
|
continue
|
||
|
|
|
||
|
|
if bool_params:
|
||
|
|
if base_key in bool_params:
|
||
|
|
params[key] = to_boolean(value)
|
||
|
|
|
||
|
|
if float_params:
|
||
|
|
if base_key in float_params:
|
||
|
|
params[key] = float(value)
|
||
|
|
|
||
|
|
if int_params:
|
||
|
|
if base_key in int_params:
|
||
|
|
params[key] = int(value)
|
||
|
|
|
||
|
|
if string_params:
|
||
|
|
if base_key in string_params:
|
||
|
|
params[key] = str(value)
|
||
|
|
|
||
|
|
if list_params:
|
||
|
|
if base_key in list_params:
|
||
|
|
params[key] = map(str, value.split(','))
|
||
|
|
|
||
|
|
if timestamp_params:
|
||
|
|
if base_key in timestamp_params:
|
||
|
|
if value.count(':') == 1:
|
||
|
|
value += ':00' # hack?
|
||
|
|
params[key] = float(datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S').strftime('%s'))# - config.TIMEZONE_OFFSET
|
||
|
|
|
||
|
|
# Consolidate composite keys
|
||
|
|
if composite_params:
|
||
|
|
for composite_key in composite_params:
|
||
|
|
aggregated_tuples = list()
|
||
|
|
for key in params:
|
||
|
|
if composite_key in key:
|
||
|
|
value = params[key]
|
||
|
|
aggregated_tuples.append((key, value))
|
||
|
|
if len(aggregated_tuples) > 0:
|
||
|
|
aggregated_tuples.sort(key=lambda tup: tup[0])
|
||
|
|
aggregated_values = [y for (_, y) in aggregated_tuples]
|
||
|
|
non_none_values = filter(lambda x: x is not None, aggregated_values)
|
||
|
|
if len(non_none_values) == len(aggregated_values):
|
||
|
|
params_json_dict[composite_key] = aggregated_values
|
||
|
|
else:
|
||
|
|
params_json_dict[composite_key] = None
|
||
|
|
else:
|
||
|
|
params_json_dict[composite_key] = None
|
||
|
|
|
||
|
|
# Consolidate custom keys
|
||
|
|
if custom_params:
|
||
|
|
for custom_key in custom_params:
|
||
|
|
aggregated_values = list()
|
||
|
|
for custom_key_field in custom_params[custom_key]:
|
||
|
|
aggregated_values.append(params[custom_key_field])
|
||
|
|
non_none_values = filter(lambda x: x is not None, aggregated_values)
|
||
|
|
if len(non_none_values) == len(aggregated_values):
|
||
|
|
params_json_dict[custom_key] = aggregated_values
|
||
|
|
else:
|
||
|
|
params_json_dict[custom_key] = None
|
||
|
|
|
||
|
|
# Copy remaining keys
|
||
|
|
for key in params:
|
||
|
|
if str(key)[-1].isdigit():
|
||
|
|
base_key = str(key)[:-2]
|
||
|
|
else:
|
||
|
|
base_key = key
|
||
|
|
|
||
|
|
if composite_params:
|
||
|
|
if base_key not in composite_params and base_key not in all_custom_key_fields:
|
||
|
|
params_json_dict[key] = params[key]
|
||
|
|
else:
|
||
|
|
params_json_dict[key] = params[key]
|
||
|
|
|
||
|
|
# Eliminate None params
|
||
|
|
params_json_dict = {k: v for k, v in params_json_dict.iteritems() if v is not None}
|
||
|
|
|
||
|
|
return params_json_dict
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/search', methods=['GET'])
|
||
|
|
def search_form():
|
||
|
|
form = SearchForm()
|
||
|
|
|
||
|
|
# Get list of host mappings
|
||
|
|
show_hosts_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/hosts/get'
|
||
|
|
hosts = map(json.loads, requests.get(show_hosts_url).text.splitlines())
|
||
|
|
|
||
|
|
# Get list of services mappings
|
||
|
|
show_services_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/services/get'
|
||
|
|
services = map(json.loads, requests.get(show_services_url).text.splitlines())
|
||
|
|
|
||
|
|
services = [(service['name'], json.dumps(service)) for service in services]
|
||
|
|
services.insert(0, ('-', '-'))
|
||
|
|
hosts = [(host['name'], json.dumps(host)) for host in hosts]
|
||
|
|
hosts.insert(0, ('-', '-'))
|
||
|
|
return render_template('search_form.html', title='Search PCAP', form=form, hosts=hosts, services=services)
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/search', methods=['POST'])
|
||
|
|
def do_search():
|
||
|
|
search_params = request.form.to_dict()
|
||
|
|
bool_params = ['search_dst', 'syn', 'synack', 'fin', 'rst']
|
||
|
|
int_params = ['dst_port']
|
||
|
|
float_params = ['src_boot', 'dst_boot', 'num_packets', 'duration',
|
||
|
|
'dst_size_sent', 'dst_printables', 'dst_nonprintables', 'src_size_sent', 'src_printables', 'src_nonprintables']
|
||
|
|
string_params = ['src_host', 'dst_host', 'search_regex', 'protocol', 'filename']
|
||
|
|
list_params = ['tags']
|
||
|
|
timestamp_params = ['timestamp']
|
||
|
|
composite_params = ['timestamp', 'dst_id', 'num_packets', 'duration',
|
||
|
|
'dst_size_sent', 'dst_printables', 'dst_nonprintables', 'src_size_sent', 'src_printables',
|
||
|
|
'src_nonprintables']
|
||
|
|
custom_params = {'dst_id': ['protocol', 'dst_host', 'dst_port']}
|
||
|
|
|
||
|
|
param_map = {'bool_params': bool_params, 'int_params': int_params, 'float_params': float_params, 'string_params': string_params,
|
||
|
|
'list_params': list_params, 'timestamp_params': timestamp_params, 'composite_params': composite_params, 'custom_params': custom_params}
|
||
|
|
search_params_json_dict = jsonify_params(search_params, **param_map)
|
||
|
|
|
||
|
|
search = Search.search(search_params_json_dict)
|
||
|
|
return redirect('/search/%d/1' % search.num)
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/search/<sid>/<page>', methods=['GET'])
|
||
|
|
def load_search(sid, page):
|
||
|
|
sorting = request.args.get('sort', 'timestamp')
|
||
|
|
reverse = request.args.get('reverse', 'no') == 'yes'
|
||
|
|
search = Search.load(sid, sorting)
|
||
|
|
page_size = 50
|
||
|
|
npages = search.num_results // page_size
|
||
|
|
if search.num_results % page_size != 0:
|
||
|
|
npages += 1
|
||
|
|
return render_template('search.html', title='Search Results', search=search, page=int(page), page_size=page_size, npages=npages, reverse=reverse, qstring=request.query_string)
|
||
|
|
|
||
|
|
@app.route('/get_metadata', methods=['GET'])
|
||
|
|
def get_metadata():
|
||
|
|
matched_ids = request.args.to_dict()
|
||
|
|
ids_to_fetch_metadata_for = []
|
||
|
|
for key, value in matched_ids.iteritems():
|
||
|
|
if value == 'on':
|
||
|
|
ids_to_fetch_metadata_for.append(int(key))
|
||
|
|
|
||
|
|
connection_metadata_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/metadata?%s' % json.dumps(ids_to_fetch_metadata_for)
|
||
|
|
connection_metadata = map(json.loads, requests.get(connection_metadata_url).text.splitlines())
|
||
|
|
return render_template('metadata.html', title='Connection Metadata', connection_metadata=connection_metadata)
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/download_pcap', methods=['GET'])
|
||
|
|
def download_pcap():
|
||
|
|
pcap_file = request.args['filename']
|
||
|
|
download_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/pcap?%s' % pcap_file
|
||
|
|
pcap_data = requests.get(download_url).content
|
||
|
|
res = make_response(pcap_data)
|
||
|
|
res.headers["Content-Disposition"] = "attachment; filename=" + pcap_file
|
||
|
|
return res
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/add_service_form', methods=['GET'])
|
||
|
|
def add_service_form():
|
||
|
|
form = AddServiceForm()
|
||
|
|
return render_template('add_service_form.html', title="Add Service", form=form)
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/add_service', methods=['GET'])
|
||
|
|
def add_service():
|
||
|
|
add_service_params = request.args.to_dict()
|
||
|
|
string_params = ['name', 'host', 'protocol']
|
||
|
|
int_params = ['port']
|
||
|
|
param_map = {'int_params': int_params, 'string_params': string_params}
|
||
|
|
add_params_json_dict = jsonify_params(add_service_params, **param_map)
|
||
|
|
|
||
|
|
add_service_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/services/set?%s' % json.dumps(add_params_json_dict)
|
||
|
|
requests.get(add_service_url)
|
||
|
|
return redirect('/show_services')
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/show_services', methods=['GET'])
|
||
|
|
def show_services():
|
||
|
|
show_services_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/services/get'
|
||
|
|
services = map(json.loads, requests.get(show_services_url).text.splitlines())
|
||
|
|
return render_template('services.html', title="Services", services=services)
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/delete_service', methods=['GET'])
|
||
|
|
def delete_service():
|
||
|
|
delete_service_params = request.args.to_dict()
|
||
|
|
string_params = ['name', 'host', 'protocol']
|
||
|
|
int_params = ['port']
|
||
|
|
param_map = {'int_params': int_params, 'string_params': string_params}
|
||
|
|
delete_params_json_dict = jsonify_params(delete_service_params, **param_map)
|
||
|
|
|
||
|
|
delete_service_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/services/del?%s' % json.dumps(delete_params_json_dict)
|
||
|
|
requests.get(delete_service_url)
|
||
|
|
return redirect('/show_services')
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/add_host_form', methods=['GET'])
|
||
|
|
def add_host_form():
|
||
|
|
form = AddHostForm()
|
||
|
|
return render_template('add_host_form.html', title="Add Host", form=form)
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/add_host', methods=['GET'])
|
||
|
|
def add_host():
|
||
|
|
add_host_params = request.args.to_dict()
|
||
|
|
string_params = ['name']
|
||
|
|
float_params = ['boot_time']
|
||
|
|
param_map = {'float_params': float_params, 'string_params': string_params}
|
||
|
|
add_params_json_dict = jsonify_params(add_host_params, **param_map)
|
||
|
|
|
||
|
|
add_host_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/hosts/set?%s' % json.dumps(add_params_json_dict)
|
||
|
|
requests.get(add_host_url)
|
||
|
|
return redirect('/show_hosts')
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/show_hosts', methods=['GET'])
|
||
|
|
def show_hosts():
|
||
|
|
show_hosts_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/hosts/get'
|
||
|
|
hosts = map(json.loads, requests.get(show_hosts_url).text.splitlines())
|
||
|
|
return render_template('hosts.html', title="Hosts", hosts=hosts)
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/delete_host', methods=['GET'])
|
||
|
|
def delete_host():
|
||
|
|
delete_host_params = request.args.to_dict()
|
||
|
|
string_params = ['name']
|
||
|
|
float_params = ['boot_time']
|
||
|
|
param_map = {'float_params': float_params, 'string_params': string_params}
|
||
|
|
delete_params_json_dict = jsonify_params(delete_host_params, **param_map)
|
||
|
|
|
||
|
|
delete_host_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/hosts/del?%s' % json.dumps(delete_params_json_dict)
|
||
|
|
requests.get(delete_host_url)
|
||
|
|
return redirect('/show_hosts')
|
||
|
|
|
||
|
|
|
||
|
|
@app.route('/connection/<cid>', methods=['GET'])
|
||
|
|
def show_connection(cid):
|
||
|
|
metadata = load_metadata([int(cid)])[0]
|
||
|
|
print 'metadata: ', metadata
|
||
|
|
pcap = local_pcap_path(metadata['filename'])
|
||
|
|
stream_data = list(parse_pcap(pcap, metadata))
|
||
|
|
return render_template('view_connection.html', md=metadata, connection=metadata, stream_data=json.dumps(stream_data), title='View Connection', hex_width=3 if config.BYTE_WIDTH == 9 else 2)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_pcap(filename, md):
|
||
|
|
filestream = open(filename, 'rb')
|
||
|
|
first_timestamp = md['timestamp']
|
||
|
|
pcap_stream = dpkt.pcap.Reader(filestream)
|
||
|
|
decode = {8: decode_octet_stream, 9: decode_nyte_stream}[config.BYTE_WIDTH]
|
||
|
|
|
||
|
|
LEFTOVER = BitStream()
|
||
|
|
for timestamp, packet in pcap_stream:
|
||
|
|
eth = dpkt.ethernet.Ethernet(packet)
|
||
|
|
# TODO: add checks for packet type
|
||
|
|
ip = eth.data
|
||
|
|
tcp = ip.data
|
||
|
|
|
||
|
|
if tcp.data == '':
|
||
|
|
continue
|
||
|
|
direction = 'recv' if tcp.sport == md['dst_port'] else 'send'
|
||
|
|
|
||
|
|
# TODO configurable MTU?
|
||
|
|
data = BitStream(bytes=tcp.data)
|
||
|
|
if LEFTOVER.len:
|
||
|
|
data = LEFTOVER + data
|
||
|
|
LEFTOVER = BitStream()
|
||
|
|
if (len(data) - len(LEFTOVER)) > 1440*8 and (len(data) % 9):
|
||
|
|
# multiple packets, content have been split
|
||
|
|
LEFTOVER = data[-(data.len % 9):]
|
||
|
|
print LEFTOVER.len
|
||
|
|
print data.len
|
||
|
|
data = data[:-LEFTOVER.len]
|
||
|
|
print data.len
|
||
|
|
to_decode = data
|
||
|
|
if (to_decode.len % 8):
|
||
|
|
# add some padding
|
||
|
|
to_decode += BitStream('0b'+'0'*(8-(data.len%8)))
|
||
|
|
|
||
|
|
yield {'direction': direction, 'timediff': timestamp-first_timestamp, 'data': decode(to_decode.bytes)}
|
||
|
|
|
||
|
|
def decode_octet_stream(data):
|
||
|
|
return map(ord, data)
|
||
|
|
|
||
|
|
def decode_nyte_stream(n):
|
||
|
|
bin_str = nytes_to_bit_string(n)
|
||
|
|
return [int(bin_str[i:i+9], 2) for i in xrange(0, len(bin_str), 9)]
|
||
|
|
|
||
|
|
def nytes_to_bit_string(n):
|
||
|
|
bin_str = "".join(bin(ord(c))[2:].zfill(8) for c in n)
|
||
|
|
num_bits = (len(n) * 8) % 9
|
||
|
|
return bin_str[:len(bin_str) - num_bits]
|