"""Reads network data from PCAP file."""

ssl_ports = 443,

notified_dpkt = False

def __init__(self, filepath):

"""Creates a new instance.

self.filepath = filepath

# List of all hosts.

self.hosts = []

# List containing all non-private IP addresses.

self.unique_hosts = []

# List of unique domains.

self.unique_domains = []

# List containing all TCP packets.

self.tcp_connections = []

self.tcp_connections_seen = set()

# List containing all UDP packets.

self.udp_connections = []

self.udp_connections_seen = set()

# List containing all ICMP requests.

self.icmp_requests = []

# List containing all HTTP requests.

self.http_requests = {}

# List containing all TLS/SSL3 key combinations.

self.tls_keys = []

# List containing all DNS requests.

self.dns_requests = {}

self.dns_answers = set()

# List containing all SMTP requests.

self.smtp_requests = []

# Reconstruncted SMTP flow.

self.smtp_flow = {}

# List containing all IRC requests.

self.irc_requests = []

# Dictionary containing all the results of this processing.

self.results = {}

def _dns_gethostbyname(self, name):

"""Get host by name wrapper.

if Config().processing.resolve_dns:

ip = resolve(name)

else:

ip = ""

return ip

def _is_private_ip(self, ip):

"""Check if the IP belongs to private network blocks.

networks = [

"0.0.0.0/8",

"10.0.0.0/8",

"100.64.0.0/10",

"127.0.0.0/8",

"169.254.0.0/16",

"172.16.0.0/12",

"192.0.0.0/24",

"192.0.2.0/24",

"192.88.99.0/24",

"192.168.0.0/16",

"198.18.0.0/15",

"198.51.100.0/24",

"203.0.113.0/24",

"240.0.0.0/4",

"255.255.255.255/32",

"224.0.0.0/4"

]

for network in networks:

try:

ipaddr = struct.unpack(">I", socket.inet_aton(ip))[0]

netaddr, bits = network.split("/")

network_low = struct.unpack(">I", socket.inet_aton(netaddr))[0]

network_high = network_low | (1 << (32 - int(bits))) - 1

if ipaddr <= network_high and ipaddr >= network_low:

return True

except:

continue

return False

def _add_hosts(self, connection):

"""Add IPs to unique list.

try:

# TODO: Perhaps this block should be removed.

# If there is a packet from a non-local IP address, which hasn't

# been seen before, it means that the connection wasn't initiated

# during the time of the current analysis.

if connection["src"] not in self.hosts:

ip = convert_to_printable(connection["src"])

# We consider the IP only if it hasn't been seen before.

if ip not in self.hosts:

# If the IP is not a local one, this might be a leftover

# packet as described in issue #249.

if self._is_private_ip(ip):

self.hosts.append(ip)

if connection["dst"] not in self.hosts:

ip = convert_to_printable(connection["dst"])

if ip not in self.hosts:

self.hosts.append(ip)

# We add external IPs to the list, only the first time

# we see them and if they're the destination of the

# first packet they appear in.

if not self._is_private_ip(ip):

self.unique_hosts.append(ip)

except:

pass

def _tcp_dissect(self, conn, data):

"""Runs all TCP dissectors.

if self._check_http(data):

self._add_http(data, conn["dport"])

# SMTP.

if conn["dport"] == 25:

self._reassemble_smtp(conn, data)

# IRC.

if conn["dport"] != 21 and self._check_irc(data):

self._add_irc(data)

# HTTPS.

if conn["dport"] in self.ssl_ports or conn["sport"] in self.ssl_ports:

self._https_identify(conn, data)

def _udp_dissect(self, conn, data):

"""Runs all UDP dissectors.

# Select DNS and MDNS traffic.

if conn["dport"] == 53 or conn["sport"] == 53 or conn["dport"] == 5353 or conn["sport"] == 5353:

if self._check_dns(data):

self._add_dns(data)

def _check_icmp(self, icmp_data):

"""Checks for ICMP traffic.

try:

return isinstance(icmp_data, dpkt.icmp.ICMP) and \

len(icmp_data.data) > 0

except:

return False

def _icmp_dissect(self, conn, data):

"""Runs all ICMP dissectors.

if self._check_icmp(data):

# If ICMP packets are coming from the host, it probably isn't

# relevant traffic, hence we can skip from reporting it.

if conn["src"] == Config().resultserver.ip:

return

entry = {}

entry["src"] = conn["src"]

entry["dst"] = conn["dst"]

entry["type"] = data.type

# Extract data from dpkg.icmp.ICMP.

try:

entry["data"] = convert_to_printable(data.data.data)

except:

entry["data"] = ""

self.icmp_requests.append(entry)

def _check_dns(self, udpdata):

"""Checks for DNS traffic.

try:

dpkt.dns.DNS(udpdata)

except:

return False

return True

def _add_dns(self, udpdata):

"""Adds a DNS data flow.

dns = dpkt.dns.DNS(udpdata)

# DNS query parsing.

query = {}

if dns.rcode == dpkt.dns.DNS_RCODE_NOERR or \

dns.qr == dpkt.dns.DNS_R or \

dns.opcode == dpkt.dns.DNS_QUERY or True:

# DNS question.

try:

q_name = dns.qd[0].name

q_type = dns.qd[0].type

except IndexError:

return False

query["request"] = q_name

if q_type == dpkt.dns.DNS_A:

query["type"] = "A"

if q_type == dpkt.dns.DNS_AAAA:

query["type"] = "AAAA"

elif q_type == dpkt.dns.DNS_CNAME:

query["type"] = "CNAME"

elif q_type == dpkt.dns.DNS_MX:

query["type"] = "MX"

elif q_type == dpkt.dns.DNS_PTR:

query["type"] = "PTR"

elif q_type == dpkt.dns.DNS_NS:

query["type"] = "NS"

elif q_type == dpkt.dns.DNS_SOA:

query["type"] = "SOA"

elif q_type == dpkt.dns.DNS_HINFO:

query["type"] = "HINFO"

elif q_type == dpkt.dns.DNS_TXT:

query["type"] = "TXT"

elif q_type == dpkt.dns.DNS_SRV:

query["type"] = "SRV"

# DNS answer.

query["answers"] = []

for answer in dns.an:

ans = {}

if answer.type == dpkt.dns.DNS_A:

ans["type"] = "A"

try:

ans["data"] = socket.inet_ntoa(answer.rdata)

except socket.error:

continue

elif answer.type == dpkt.dns.DNS_AAAA:

ans["type"] = "AAAA"

try:

ans["data"] = socket.inet_ntop(socket.AF_INET6,

answer.rdata)

except (socket.error, ValueError):

continue

elif answer.type == dpkt.dns.DNS_CNAME:

ans["type"] = "CNAME"

ans["data"] = answer.cname

elif answer.type == dpkt.dns.DNS_MX:

ans["type"] = "MX"

ans["data"] = answer.mxname

elif answer.type == dpkt.dns.DNS_PTR:

ans["type"] = "PTR"

ans["data"] = answer.ptrname

elif answer.type == dpkt.dns.DNS_NS:

ans["type"] = "NS"

ans["data"] = answer.nsname

elif answer.type == dpkt.dns.DNS_SOA:

ans["type"] = "SOA"

ans["data"] = ",".join([answer.mname,

answer.rname,

str(answer.serial),

str(answer.refresh),

str(answer.retry),

str(answer.expire),

str(answer.minimum)])

elif answer.type == dpkt.dns.DNS_HINFO:

ans["type"] = "HINFO"

ans["data"] = " ".join(answer.text)

elif answer.type == dpkt.dns.DNS_TXT:

ans["type"] = "TXT"

ans["data"] = " ".join(answer.text)

# TODO: add srv handling

query["answers"].append(ans)

self._add_domain(query["request"])

reqtuple = query["type"], query["request"]

if reqtuple not in self.dns_requests:

self.dns_requests[reqtuple] = query

else:

new_answers = set((i["type"], i["data"]) for i in query["answers"]) - self.dns_answers

self.dns_requests[reqtuple]["answers"] += [dict(type=i[0], data=i[1]) for i in new_answers]

return True

def _add_domain(self, domain):

"""Add a domain to unique list.

filters = [

".*\\.windows\\.com$",

".*\\.in\\-addr\\.arpa$"

]

regexps = [re.compile(filter) for filter in filters]

for regexp in regexps:

if regexp.match(domain):

return

for entry in self.unique_domains:

if entry["domain"] == domain:

return

self.unique_domains.append({"domain": domain,

"ip": self._dns_gethostbyname(domain)})

def _check_http(self, tcpdata):

"""Checks for HTTP traffic.

try:

r = dpkt.http.Request()

r.method, r.version, r.uri = None, None, None

r.unpack(tcpdata)

except dpkt.dpkt.UnpackError:

if r.method is not None or r.version is not None or \

r.uri is not None:

return True

return False

return True

def _add_http(self, tcpdata, dport):

"""Adds an HTTP flow.

if tcpdata in self.http_requests:

self.http_requests[tcpdata]["count"] += 1

return True

try:

http = dpkt.http.Request()

http.unpack(tcpdata)

except dpkt.dpkt.UnpackError:

pass

try:

entry = {"count": 1}

if "host" in http.headers:

entry["host"] = convert_to_printable(http.headers["host"])

else:

entry["host"] = ""

entry["port"] = dport

# Manually deal with cases when destination port is not the

# default one and it is not included in host header.

netloc = entry["host"]

if dport != 80 and ":" not in netloc:

netloc += ":" + str(entry["port"])

entry["data"] = convert_to_printable(tcpdata)

url = urlparse.urlunparse(("http", netloc, http.uri,

None, None, None))

entry["uri"] = convert_to_printable(url)

entry["body"] = convert_to_printable(http.body)

entry["path"] = convert_to_printable(http.uri)

if "user-agent" in http.headers:

entry["user-agent"] = \

convert_to_printable(http.headers["user-agent"])

entry["version"] = convert_to_printable(http.version)

entry["method"] = convert_to_printable(http.method)

self.http_requests[tcpdata] = entry

except Exception:

return False

return True

def _https_identify(self, conn, data):

"""Extract a combination of the Session ID, Client Random, and Server

if not hasattr(dpkt.ssl, "TLSRecord"):

if not Pcap.notified_dpkt:

Pcap.notified_dpkt = True

log.warning("Using an old version of dpkt that does not "

"support TLS streams (install the latest with "

"`pip install dpkt`)")

return

try:

record = dpkt.ssl.TLSRecord(data)

except dpkt.NeedData:

return

except:

log.exception("Error reading possible TLS Record")

return

# Is this a valid TLS packet?

if record.type not in dpkt.ssl.RECORD_TYPES:

return

try:

record = dpkt.ssl.RECORD_TYPES[record.type](record.data)

except dpkt.ssl.SSL3Exception:

return

except dpkt.NeedData:

log.exception("Incomplete possible TLS Handshake record found")

return

# Is this a TLSv1 Handshake packet?

if not isinstance(record, dpkt.ssl.TLSHandshake):

return

# We're only interested in the TLS Server Hello packets.

if not isinstance(record.data, dpkt.ssl.TLSServerHello):

return

# Extract the server random and the session id.

self.tls_keys.append({

"server_random": record.data.random.encode("hex"),

"session_id": record.data.session_id.encode("hex"),

})

def _reassemble_smtp(self, conn, data):

"""Reassemble a SMTP flow.

if conn["dst"] in self.smtp_flow:

self.smtp_flow[conn["dst"]] += data

else:

self.smtp_flow[conn["dst"]] = data

def _process_smtp(self):

"""Process SMTP flow."""

for conn, data in self.smtp_flow.iteritems():

# Detect new SMTP flow.

if data.startswith(("EHLO", "HELO")):

self.smtp_requests.append({"dst": conn, "raw": data})

def _check_irc(self, tcpdata):

"""

try:

req = ircMessage()

except Exception:

return False

return req.isthereIRC(tcpdata)

def _add_irc(self, tcpdata):

"""

try:

reqc = ircMessage()

reqs = ircMessage()

filters_sc = ["266"]

self.irc_requests = self.irc_requests + \

reqc.getClientMessages(tcpdata) + \

reqs.getServerMessagesFilter(tcpdata, filters_sc)

except Exception:

return False

return True

def run(self):

"""Process PCAP.

try:

file = open(self.filepath, "rb")

except (IOError, OSError):

log.error("Unable to open %s" % self.filepath)

return self.results

try:

pcap = dpkt.pcap.Reader(file)

except dpkt.dpkt.NeedData:

log.error("Unable to read PCAP file at path \"%s\".",

self.filepath)

return self.results

except ValueError:

log.error("Unable to read PCAP file at path \"%s\". File is "

"corrupted or wrong format." % self.filepath)

return self.results

offset = file.tell()

first_ts = None

for ts, buf in pcap:

if not first_ts:

first_ts = ts

try:

ip = iplayer_from_raw(buf, pcap.datalink())

connection = {}

if isinstance(ip, dpkt.ip.IP):

connection["src"] = socket.inet_ntoa(ip.src)

connection["dst"] = socket.inet_ntoa(ip.dst)

elif isinstance(ip, dpkt.ip6.IP6):

connection["src"] = socket.inet_ntop(socket.AF_INET6,

ip.src)

connection["dst"] = socket.inet_ntop(socket.AF_INET6,

ip.dst)

else:

offset = file.tell()

continue

self._add_hosts(connection)

if ip.p == dpkt.ip.IP_PROTO_TCP:

tcp = ip.data

if not isinstance(tcp, dpkt.tcp.TCP):

tcp = dpkt.tcp.TCP(tcp)

if tcp.data:

connection["sport"] = tcp.sport

connection["dport"] = tcp.dport

self._tcp_dissect(connection, tcp.data)

src, sport, dst, dport = (connection["src"], connection["sport"], connection["dst"], connection["dport"])

if not ((dst, dport, src, sport) in self.tcp_connections_seen or (src, sport, dst, dport) in self.tcp_connections_seen):

self.tcp_connections.append((src, sport, dst, dport, offset, ts-first_ts))

self.tcp_connections_seen.add((src, sport, dst, dport))

elif ip.p == dpkt.ip.IP_PROTO_UDP:

udp = ip.data

if not isinstance(udp, dpkt.udp.UDP):

udp = dpkt.udp.UDP(udp)

if len(udp.data) > 0:

connection["sport"] = udp.sport

connection["dport"] = udp.dport

self._udp_dissect(connection, udp.data)

src, sport, dst, dport = (connection["src"], connection["sport"], connection["dst"], connection["dport"])

if not ((dst, dport, src, sport) in self.udp_connections_seen or (src, sport, dst, dport) in self.udp_connections_seen):

self.udp_connections.append((src, sport, dst, dport, offset, ts-first_ts))

self.udp_connections_seen.add((src, sport, dst, dport))

elif ip.p == dpkt.ip.IP_PROTO_ICMP:

icmp = ip.data

if not isinstance(icmp, dpkt.icmp.ICMP):

icmp = dpkt.icmp.ICMP(icmp)

self._icmp_dissect(connection, icmp)

offset = file.tell()

except AttributeError:

continue

except dpkt.dpkt.NeedData:

continue

except Exception as e:

log.exception("Failed to process packet: %s", e)

file.close()

# Post processors for reconstructed flows.

self._process_smtp()

# Build results dict.

self.results["hosts"] = self.unique_hosts

self.results["domains"] = self.unique_domains

self.results["tcp"] = [conn_from_flowtuple(i) for i in self.tcp_connections]

self.results["udp"] = [conn_from_flowtuple(i) for i in self.udp_connections]

self.results["icmp"] = self.icmp_requests

self.results["http"] = self.http_requests.values()

self.results["tls"] = self.tls_keys

self.results["dns"] = self.dns_requests.values()

self.results["smtp"] = self.smtp_requests

self.results["irc"] = self.irc_requests

"""Network analysis."""

def run(self):

self.key = "network"

results = {}

# Include any results provided by the mitm script.

results["mitm"] = []

if os.path.exists(self.mitmout_path):

for line in open(self.mitmout_path, "rb"):

try:

results["mitm"].append(json.loads(line))

except:

results["mitm"].append(line)

if not IS_DPKT:

log.error("Python DPKT is not installed, aborting PCAP analysis.")

return results

if not os.path.exists(self.pcap_path):

log.warning("The PCAP file does not exist at path \"%s\".",

self.pcap_path)

return results

if os.path.getsize(self.pcap_path) == 0:

log.error("The PCAP file at path \"%s\" is empty." % self.pcap_path)

return results

sorted_path = self.pcap_path.replace("dump.", "dump_sorted.")

if Config().processing.sort_pcap:

sort_pcap(self.pcap_path, sorted_path)

results.update(Pcap(sorted_path).run())

else:

results.update(Pcap(self.pcap_path).run())

# Save PCAP file hash.

if os.path.exists(self.pcap_path):

results["pcap_sha256"] = File(self.pcap_path).get_sha256()

if os.path.exists(sorted_path):

results["sorted_pcap_sha256"] = File(sorted_path).get_sha256()

"""Converts a raw packet to a dpkt packet regarding of link type.

if linktype == 1: # ethernet

pkt = dpkt.ethernet.Ethernet(raw)

ip = pkt.data

elif linktype == 101: # raw

ip = dpkt.ip.IP(raw)

else:

raise CuckooProcessingError("unknown PCAP linktype")

"""Convert the flow tuple into a dictionary (suitable for JSON)"""

sip, sport, dip, dport, offset, relts = ft

return {"src": sip, "sport": sport,

"dst": dip, "dport": dport,

"""batch sort helper with temporary files, supports sorting large stuff"""

if not output_class:

output_class = input_iterator.__class__

chunks = []

try:

while True:

current_chunk = list(islice(input_iterator, buffer_size))

if not current_chunk:

break

current_chunk.sort()

fd, filepath = tempfile.mkstemp()

os.close(fd)

output_chunk = output_class(filepath)

chunks.append(output_chunk)

for elem in current_chunk:

output_chunk.write(elem.obj)

output_chunk.close()

output_file = output_class(output_path)

for elem in heapq.merge(*chunks):

output_file.write(elem.obj)

output_file.close()

finally:

for chunk in chunks:

try:

chunk.close()

os.remove(chunk.name)

except Exception:

"""SortCap is a wrapper around the packet lib (dpkt) that allows us to sort pcaps

def __init__(self, path, linktype=1):

self.name = path

self.linktype = linktype

self.fd = None

self.ctr = 0 # counter to pass through packets without flow info (non-IP)

self.conns = set()

def write(self, p):

if not self.fd:

self.fd = dpkt.pcap.Writer(open(self.name, "wb"), linktype=self.linktype)

self.fd.writepkt(p.raw, p.ts)

def __iter__(self):

if not self.fd:

self.fd = dpkt.pcap.Reader(open(self.name, "rb"))

self.fditer = iter(self.fd)

self.linktype = self.fd.datalink()

return self

def close(self):

if self.fd:

self.fd.close()

self.fd = None

def next(self):

rp = next(self.fditer)

if rp is None:

return None

self.ctr += 1

ts, raw = rp

rpkt = Packet(raw, ts)

sip, dip, sport, dport, proto = flowtuple_from_raw(raw, self.linktype)

# check other direction of same flow

if (dip, sip, dport, sport, proto) in self.conns:

flowtuple = (dip, sip, dport, sport, proto)

else:

flowtuple = (sip, dip, sport, dport, proto)

self.conns.add(flowtuple)

"""Use SortCap class together with batch_sort to sort a pcap"""

inc = SortCap(inpath)

batch_sort(inc, outpath, output_class=lambda path: SortCap(path, linktype=inc.linktype))

"""Parse a packet from a pcap just enough to gain a flow description tuple"""

ip = iplayer_from_raw(raw, linktype)

if isinstance(ip, dpkt.ip.IP):

sip, dip = socket.inet_ntoa(ip.src), socket.inet_ntoa(ip.dst)

proto = ip.p

if proto == dpkt.ip.IP_PROTO_TCP or proto == dpkt.ip.IP_PROTO_UDP:

l3 = ip.data

sport, dport = l3.sport, l3.dport

else:

sport, dport = 0, 0

else:

sip, dip, proto = 0, 0, -1

sport, dport = 0, 0

flowtuple = (sip, dip, sport, dport, proto)

"""Get the payload from a packet, the data below TCP/UDP basically"""

ip = iplayer_from_raw(raw, linktype)

try:

return ip.data.data

except:

"""Extract all packets belonging to the same flow from a pcap packet iterator"""

first_ft = None

for ts, raw in piter:

ft = flowtuple_from_raw(raw, linktype)

if not first_ft:

first_ft = ft

sip, dip, sport, dport, proto = ft

if not (first_ft == ft or first_ft == (dip, sip, dport, sport, proto)):

break

yield {

"src": sip, "dst": dip, "sport": sport, "dport": dport,

"raw": payload_from_raw(raw, linktype).encode("base64"),

"direction": first_ft == ft,

"""Open a PCAP, seek to a packet offset, then get all packets belonging to the same connection"""

pcap = dpkt.pcap.Reader(fobj)

pcapiter = iter(pcap)

ts, raw = pcapiter.next()

fobj.seek(offset)

for p in next_connection_packets(pcapiter, linktype=pcap.datalink()):