import requests import json import config import dpkt import datetime import pprint from bitstring import BitStream from flask import render_template, request, make_response, redirect from . import app from .search_form import SearchForm from .add_service_form import AddServiceForm from .add_host_form import AddHostForm from .data import Search, load_metadata, local_pcap_path def cutoff(string, maximum): if len(string) > maximum - 3: string = string[:maximum] + '...' return string app.jinja_env.globals.update(tz_offset=config.TIMEZONE_OFFSET) app.jinja_env.globals.update(load_metadata=load_metadata) app.jinja_env.globals.update(pprint=pprint.pformat) app.jinja_env.globals.update(cutoff=cutoff) app.jinja_env.globals.update(json=json) @app.route('/') @app.route('/history') def show_history(): all_searches = sorted(Search.loadall(partial=True), key=lambda x: x.num, reverse=True) return render_template('search_history.html', searches=all_searches, title='Search History') def to_boolean(value): if value == '1': return True elif value == '2': return False else: return None def jsonify_params(params, bool_params=None, int_params=None, float_params=None, string_params=None, list_params=None, timestamp_params=None, composite_params=None, custom_params=None): params_json_dict = {} all_custom_key_fields = list() if custom_params: for custom_key in custom_params: all_custom_key_fields.extend(custom_params[custom_key]) # Sanitize and convert parameters for key, value in params.iteritems(): if str(key)[-1].isdigit(): base_key = str(key)[:-2] else: base_key = key value = params[key] if value == '': params[key] = None continue if bool_params: if base_key in bool_params: params[key] = to_boolean(value) if float_params: if base_key in float_params: params[key] = float(value) if int_params: if base_key in int_params: params[key] = int(value) if string_params: if base_key in string_params: params[key] = str(value) if list_params: if base_key in list_params: params[key] = map(str, value.split(',')) if timestamp_params: if base_key in timestamp_params: if value.count(':') == 1: value += ':00' # hack? params[key] = float(datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S').strftime('%s'))# - config.TIMEZONE_OFFSET # Consolidate composite keys if composite_params: for composite_key in composite_params: aggregated_tuples = list() for key in params: if composite_key in key: value = params[key] aggregated_tuples.append((key, value)) if len(aggregated_tuples) > 0: aggregated_tuples.sort(key=lambda tup: tup[0]) aggregated_values = [y for (_, y) in aggregated_tuples] non_none_values = filter(lambda x: x is not None, aggregated_values) if len(non_none_values) == len(aggregated_values): params_json_dict[composite_key] = aggregated_values else: params_json_dict[composite_key] = None else: params_json_dict[composite_key] = None # Consolidate custom keys if custom_params: for custom_key in custom_params: aggregated_values = list() for custom_key_field in custom_params[custom_key]: aggregated_values.append(params[custom_key_field]) non_none_values = filter(lambda x: x is not None, aggregated_values) if len(non_none_values) == len(aggregated_values): params_json_dict[custom_key] = aggregated_values else: params_json_dict[custom_key] = None # Copy remaining keys for key in params: if str(key)[-1].isdigit(): base_key = str(key)[:-2] else: base_key = key if composite_params: if base_key not in composite_params and base_key not in all_custom_key_fields: params_json_dict[key] = params[key] else: params_json_dict[key] = params[key] # Eliminate None params params_json_dict = {k: v for k, v in params_json_dict.iteritems() if v is not None} return params_json_dict @app.route('/search', methods=['GET']) def search_form(): form = SearchForm() # Get list of host mappings show_hosts_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/hosts/get' hosts = map(json.loads, requests.get(show_hosts_url).text.splitlines()) # Get list of services mappings show_services_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/services/get' services = map(json.loads, requests.get(show_services_url).text.splitlines()) services = [(service['name'], json.dumps(service)) for service in services] services.insert(0, ('-', '-')) hosts = [(host['name'], json.dumps(host)) for host in hosts] hosts.insert(0, ('-', '-')) return render_template('search_form.html', title='Search PCAP', form=form, hosts=hosts, services=services) @app.route('/search', methods=['POST']) def do_search(): search_params = request.form.to_dict() bool_params = ['search_dst', 'syn', 'synack', 'fin', 'rst'] int_params = ['dst_port'] float_params = ['src_boot', 'dst_boot', 'num_packets', 'duration', 'dst_size_sent', 'dst_printables', 'dst_nonprintables', 'src_size_sent', 'src_printables', 'src_nonprintables'] string_params = ['src_host', 'dst_host', 'search_regex', 'protocol', 'filename'] list_params = ['tags'] timestamp_params = ['timestamp'] composite_params = ['timestamp', 'dst_id', 'num_packets', 'duration', 'dst_size_sent', 'dst_printables', 'dst_nonprintables', 'src_size_sent', 'src_printables', 'src_nonprintables'] custom_params = {'dst_id': ['protocol', 'dst_host', 'dst_port']} param_map = {'bool_params': bool_params, 'int_params': int_params, 'float_params': float_params, 'string_params': string_params, 'list_params': list_params, 'timestamp_params': timestamp_params, 'composite_params': composite_params, 'custom_params': custom_params} search_params_json_dict = jsonify_params(search_params, **param_map) search = Search.search(search_params_json_dict) return redirect('/search/%d/1' % search.num) @app.route('/search//', methods=['GET']) def load_search(sid, page): sorting = request.args.get('sort', 'timestamp') reverse = request.args.get('reverse', 'no') == 'yes' search = Search.load(sid, sorting) page_size = 50 npages = search.num_results // page_size if search.num_results % page_size != 0: npages += 1 return render_template('search.html', title='Search Results', search=search, page=int(page), page_size=page_size, npages=npages, reverse=reverse, qstring=request.query_string) @app.route('/get_metadata', methods=['GET']) def get_metadata(): matched_ids = request.args.to_dict() ids_to_fetch_metadata_for = [] for key, value in matched_ids.iteritems(): if value == 'on': ids_to_fetch_metadata_for.append(int(key)) connection_metadata_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/metadata?%s' % json.dumps(ids_to_fetch_metadata_for) connection_metadata = map(json.loads, requests.get(connection_metadata_url).text.splitlines()) return render_template('metadata.html', title='Connection Metadata', connection_metadata=connection_metadata) @app.route('/download_pcap', methods=['GET']) def download_pcap(): pcap_file = request.args['filename'] download_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/pcap?%s' % pcap_file pcap_data = requests.get(download_url).content res = make_response(pcap_data) res.headers["Content-Disposition"] = "attachment; filename=" + pcap_file return res @app.route('/add_service_form', methods=['GET']) def add_service_form(): form = AddServiceForm() return render_template('add_service_form.html', title="Add Service", form=form) @app.route('/add_service', methods=['GET']) def add_service(): add_service_params = request.args.to_dict() string_params = ['name', 'host', 'protocol'] int_params = ['port'] param_map = {'int_params': int_params, 'string_params': string_params} add_params_json_dict = jsonify_params(add_service_params, **param_map) add_service_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/services/set?%s' % json.dumps(add_params_json_dict) requests.get(add_service_url) return redirect('/show_services') @app.route('/show_services', methods=['GET']) def show_services(): show_services_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/services/get' services = map(json.loads, requests.get(show_services_url).text.splitlines()) return render_template('services.html', title="Services", services=services) @app.route('/delete_service', methods=['GET']) def delete_service(): delete_service_params = request.args.to_dict() string_params = ['name', 'host', 'protocol'] int_params = ['port'] param_map = {'int_params': int_params, 'string_params': string_params} delete_params_json_dict = jsonify_params(delete_service_params, **param_map) delete_service_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/services/del?%s' % json.dumps(delete_params_json_dict) requests.get(delete_service_url) return redirect('/show_services') @app.route('/add_host_form', methods=['GET']) def add_host_form(): form = AddHostForm() return render_template('add_host_form.html', title="Add Host", form=form) @app.route('/add_host', methods=['GET']) def add_host(): add_host_params = request.args.to_dict() string_params = ['name'] float_params = ['boot_time'] param_map = {'float_params': float_params, 'string_params': string_params} add_params_json_dict = jsonify_params(add_host_params, **param_map) add_host_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/hosts/set?%s' % json.dumps(add_params_json_dict) requests.get(add_host_url) return redirect('/show_hosts') @app.route('/show_hosts', methods=['GET']) def show_hosts(): show_hosts_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/hosts/get' hosts = map(json.loads, requests.get(show_hosts_url).text.splitlines()) return render_template('hosts.html', title="Hosts", hosts=hosts) @app.route('/delete_host', methods=['GET']) def delete_host(): delete_host_params = request.args.to_dict() string_params = ['name'] float_params = ['boot_time'] param_map = {'float_params': float_params, 'string_params': string_params} delete_params_json_dict = jsonify_params(delete_host_params, **param_map) delete_host_url = 'http://' + config.SERVER_IP + ':' + str(config.SERVER_PORT) + '/api/hosts/del?%s' % json.dumps(delete_params_json_dict) requests.get(delete_host_url) return redirect('/show_hosts') @app.route('/connection/', methods=['GET']) def show_connection(cid): metadata = load_metadata([int(cid)])[0] print 'metadata: ', metadata pcap = local_pcap_path(metadata['filename']) stream_data = list(parse_pcap(pcap, metadata)) return render_template('view_connection.html', md=metadata, connection=metadata, stream_data=json.dumps(stream_data), title='View Connection', hex_width=3 if config.BYTE_WIDTH == 9 else 2) def parse_pcap(filename, md): filestream = open(filename, 'rb') first_timestamp = md['timestamp'] pcap_stream = dpkt.pcap.Reader(filestream) decode = {8: decode_octet_stream, 9: decode_nyte_stream}[config.BYTE_WIDTH] LEFTOVER = BitStream() for timestamp, packet in pcap_stream: eth = dpkt.ethernet.Ethernet(packet) # TODO: add checks for packet type ip = eth.data tcp = ip.data if tcp.data == '': continue direction = 'recv' if tcp.sport == md['dst_port'] else 'send' # TODO configurable MTU? data = BitStream(bytes=tcp.data) if LEFTOVER.len: data = LEFTOVER + data LEFTOVER = BitStream() if (len(data) - len(LEFTOVER)) > 1440*8 and (len(data) % 9): # multiple packets, content have been split LEFTOVER = data[-(data.len % 9):] print LEFTOVER.len print data.len data = data[:-LEFTOVER.len] print data.len to_decode = data if (to_decode.len % 8): # add some padding to_decode += BitStream('0b'+'0'*(8-(data.len%8))) yield {'direction': direction, 'timediff': timestamp-first_timestamp, 'data': decode(to_decode.bytes)} def decode_octet_stream(data): return map(ord, data) def decode_nyte_stream(n): bin_str = nytes_to_bit_string(n) return [int(bin_str[i:i+9], 2) for i in xrange(0, len(bin_str), 9)] def nytes_to_bit_string(n): bin_str = "".join(bin(ord(c))[2:].zfill(8) for c in n) num_bits = (len(n) * 8) % 9 return bin_str[:len(bin_str) - num_bits]