"""Reads network data from PCAP file."""

def __init__(self, filepath):

"""Creates a new instance.

self.filepath = filepath

# List of all hosts.

self.hosts = []

# List containing all non-private IP addresses.

self.unique_hosts = []

# List of unique domains.

self.unique_domains = []

# List containing all TCP packets.

self.tcp_connections = []

# List containing all UDP packets.

self.udp_connections = []

# List containing all ICMP requests.

self.icmp_requests = []

# List containing all HTTP requests.

self.http_requests = []

# List containing all DNS requests.

self.dns_requests = []

# List containing all SMTP requests.

self.smtp_requests = []

# Reconstruncted SMTP flow.

self.smtp_flow = {}

# List containing all IRC requests.

self.irc_requests = []

# Dictionary containing all the results of this processing.

self.results = {}

def _dns_gethostbyname(self, name):

"""Get host by name wrapper.

if Config().processing.resolve_dns:

ip = resolve(name)

else:

ip = ""

return ip

def _is_private_ip(self, ip):

"""Check if the IP belongs to private network blocks.

networks = [

"0.0.0.0/8",

"10.0.0.0/8",

"100.64.0.0/10",

"127.0.0.0/8",

"169.254.0.0/16",

"172.16.0.0/12",

"192.0.0.0/24",

"192.0.2.0/24",

"192.88.99.0/24",

"192.168.0.0/16",

"198.18.0.0/15",

"198.51.100.0/24",

"203.0.113.0/24",

"240.0.0.0/4",

"255.255.255.255/32",

"224.0.0.0/4"

]

for network in networks:

try:

ipaddr = struct.unpack(">I", socket.inet_aton(ip))[0]

netaddr, bits = network.split("/")

network_low = struct.unpack(">I", socket.inet_aton(netaddr))[0]

network_high = network_low | 1 << (32 - int(bits)) - 1

if ipaddr <= network_high and ipaddr >= network_low:

return True

except:

continue

return False

def _add_hosts(self, connection):

"""Add IPs to unique list.

try:

if connection["src"] not in self.hosts:

ip = convert_to_printable(connection["src"])

if ip in self.hosts:

return

else:

self.hosts.append(ip)

if not self._is_private_ip(ip):

self.unique_hosts.append(ip)

if connection["dst"] not in self.hosts:

ip = convert_to_printable(connection["dst"])

if ip in self.hosts:

return

else:

self.hosts.append(ip)

if not self._is_private_ip(ip):

self.unique_hosts.append(ip)

except:

pass

def _tcp_dissect(self, conn, data):

"""Runs all TCP dissectors.

if self._check_http(data):

self._add_http(data, conn["dport"])

# SMTP.

if conn["dport"] == 25:

self._reassemble_smtp(conn, data)

# IRC.

if conn["dport"] != 21 and self._check_irc(data):

self._add_irc(data)

def _udp_dissect(self, conn, data):

"""Runs all UDP dissectors.

if conn["dport"] == 53 or conn["sport"] == 53:

if self._check_dns(data):

self._add_dns(data)

def _check_icmp(self, icmp_data):

"""Checks for ICMP traffic.

try:

return isinstance(icmp_data, dpkt.icmp.ICMP) and \

len(icmp_data.data) > 0

except:

return False

def _icmp_dissect(self, conn, data):

"""Runs all ICMP dissectors.

if self._check_icmp(data):

# If ICMP packets are coming from the host, it probably isn't

# relevant traffic, hence we can skip from reporting it.

if conn["src"] == Config().resultserver.ip:

return

entry = {}

entry["src"] = conn["src"]

entry["dst"] = conn["dst"]

entry["type"] = data.type

# Extract data from dpkg.icmp.ICMP.

try:

entry["data"] = convert_to_printable(data.data.data)

except:

entry["data"] = ""

self.icmp_requests.append(entry)

def _check_dns(self, udpdata):

"""Checks for DNS traffic.

try:

dpkt.dns.DNS(udpdata)

except:

return False

return True

def _add_dns(self, udpdata):

"""Adds a DNS data flow.

dns = dpkt.dns.DNS(udpdata)

# DNS query parsing.

query = {}

if dns.rcode == dpkt.dns.DNS_RCODE_NOERR or \

dns.qr == dpkt.dns.DNS_R or \

dns.opcode == dpkt.dns.DNS_QUERY or True:

# DNS question.

try:

q_name = dns.qd[0].name

q_type = dns.qd[0].type

except IndexError:

return False

query["request"] = q_name

if q_type == dpkt.dns.DNS_A:

query["type"] = "A"

if q_type == dpkt.dns.DNS_AAAA:

query["type"] = "AAAA"

elif q_type == dpkt.dns.DNS_CNAME:

query["type"] = "CNAME"

elif q_type == dpkt.dns.DNS_MX:

query["type"] = "MX"

elif q_type == dpkt.dns.DNS_PTR:

query["type"] = "PTR"

elif q_type == dpkt.dns.DNS_NS:

query["type"] = "NS"

elif q_type == dpkt.dns.DNS_SOA:

query["type"] = "SOA"

elif q_type == dpkt.dns.DNS_HINFO:

query["type"] = "HINFO"

elif q_type == dpkt.dns.DNS_TXT:

query["type"] = "TXT"

elif q_type == dpkt.dns.DNS_SRV:

query["type"] = "SRV"

# DNS answer.

query["answers"] = []

for answer in dns.an:

ans = {}

if answer.type == dpkt.dns.DNS_A:

ans["type"] = "A"

try:

ans["data"] = socket.inet_ntoa(answer.rdata)

except socket.error:

continue

elif answer.type == dpkt.dns.DNS_AAAA:

ans["type"] = "AAAA"

try:

ans["data"] = socket.inet_ntop(socket.AF_INET6,

answer.rdata)

except (socket.error, ValueError):

continue

elif answer.type == dpkt.dns.DNS_CNAME:

ans["type"] = "CNAME"

ans["data"] = answer.cname

elif answer.type == dpkt.dns.DNS_MX:

ans["type"] = "MX"

ans["data"] = answer.mxname

elif answer.type == dpkt.dns.DNS_PTR:

ans["type"] = "PTR"

ans["data"] = answer.ptrname

elif answer.type == dpkt.dns.DNS_NS:

ans["type"] = "NS"

ans["data"] = answer.nsname

elif answer.type == dpkt.dns.DNS_SOA:

ans["type"] = "SOA"

ans["data"] = ",".join([answer.mname,

answer.rname,

str(answer.serial),

str(answer.refresh),

str(answer.retry),

str(answer.expire),

str(answer.minimum)])

elif answer.type == dpkt.dns.DNS_HINFO:

ans["type"] = "HINFO"

ans["data"] = " ".join(answer.text)

elif answer.type == dpkt.dns.DNS_TXT:

ans["type"] = "TXT"

ans["data"] = " ".join(answer.text)

# TODO: add srv handling

query["answers"].append(ans)

self._add_domain(query["request"])

self.dns_requests.append(query)

return True

def _add_domain(self, domain):

"""Add a domain to unique list.

filters = [

".*\\.windows\\.com$",

".*\\.in\\-addr\\.arpa$"

]

regexps = [re.compile(filter) for filter in filters]

for regexp in regexps:

if regexp.match(domain):

return

for entry in self.unique_domains:

if entry["domain"] == domain:

return

self.unique_domains.append({"domain": domain,

"ip": self._dns_gethostbyname(domain)})

def _check_http(self, tcpdata):

"""Checks for HTTP traffic.

try:

r = dpkt.http.Request()

r.method, r.version, r.uri = None, None, None

r.unpack(tcpdata)

except dpkt.dpkt.UnpackError:

if not r.method is None or not r.version is None or \

not r.uri is None:

return True

return False

return True

def _add_http(self, tcpdata, dport):

"""Adds an HTTP flow.

try:

http = dpkt.http.Request()

http.unpack(tcpdata)

except dpkt.dpkt.UnpackError:

pass

try:

entry = {}

if "host" in http.headers:

entry["host"] = convert_to_printable(http.headers["host"])

else:

entry["host"] = ""

entry["port"] = dport

entry["data"] = convert_to_printable(tcpdata)

entry["uri"] = convert_to_printable(urlunparse(("http",

entry["host"],

http.uri, None,

None, None)))

entry["body"] = convert_to_printable(http.body)

entry["path"] = convert_to_printable(http.uri)

if "user-agent" in http.headers:

entry["user-agent"] = \

convert_to_printable(http.headers["user-agent"])

entry["version"] = convert_to_printable(http.version)

entry["method"] = convert_to_printable(http.method)

self.http_requests.append(entry)

except Exception:

return False

return True

def _reassemble_smtp(self, conn, data):

"""Reassemble a SMTP flow.

if conn["dst"] in self.smtp_flow:

self.smtp_flow[conn["dst"]] += data

else:

self.smtp_flow[conn["dst"]] = data

def _process_smtp(self):

"""Process SMTP flow."""

for conn, data in self.smtp_flow.iteritems():

# Detect new SMTP flow.

if data.startswith("EHLO") or data.startswith("HELO"):

self.smtp_requests.append({"dst": conn, "raw": data})

def _check_irc(self, tcpdata):

"""

try:

req = ircMessage()

except Exception:

return False

return req.isthereIRC(tcpdata)

def _add_irc(self, tcpdata):

"""

try:

reqc = ircMessage()

reqs = ircMessage()

filters_sc = ["266"]

self.irc_requests = self.irc_requests + \

reqc.getClientMessages(tcpdata) + \

reqs.getServerMessagesFilter(tcpdata, filters_sc)

except Exception:

return False

return True

def run(self):

"""Process PCAP.

log = logging.getLogger("Processing.Pcap")

if not IS_DPKT:

log.error("Python DPKT is not installed, aborting PCAP analysis.")

return self.results

if not os.path.exists(self.filepath):

log.warning("The PCAP file does not exist at path \"%s\".",

self.filepath)

return self.results

if os.path.getsize(self.filepath) == 0:

log.error("The PCAP file at path \"%s\" is empty." % self.filepath)

return self.results

try:

file = open(self.filepath, "rb")

except (IOError, OSError):

log.error("Unable to open %s" % self.filepath)

return self.results

try:

pcap = dpkt.pcap.Reader(file)

except dpkt.dpkt.NeedData:

log.error("Unable to read PCAP file at path \"%s\".",

self.filepath)

return self.results

except ValueError:

log.error("Unable to read PCAP file at path \"%s\". File is "

"corrupted or wrong format." % self.filepath)

return self.results

for ts, buf in pcap:

try:

eth = dpkt.ethernet.Ethernet(buf)

ip = eth.data

connection = {}

if isinstance(ip, dpkt.ip.IP):

connection["src"] = socket.inet_ntoa(ip.src)

connection["dst"] = socket.inet_ntoa(ip.dst)

elif isinstance(ip, dpkt.ip6.IP6):

connection["src"] = socket.inet_ntop(socket.AF_INET6,

ip.src)

connection["dst"] = socket.inet_ntop(socket.AF_INET6,

ip.dst)

else:

continue

self._add_hosts(connection)

if ip.p == dpkt.ip.IP_PROTO_TCP:

tcp = ip.data

if len(tcp.data) > 0:

connection["sport"] = tcp.sport

connection["dport"] = tcp.dport

self._tcp_dissect(connection, tcp.data)

self.tcp_connections.append(connection)

else:

continue

elif ip.p == dpkt.ip.IP_PROTO_UDP:

udp = ip.data

if len(udp.data) > 0:

connection["sport"] = udp.sport

connection["dport"] = udp.dport

self._udp_dissect(connection, udp.data)

self.udp_connections.append(connection)

elif ip.p == dpkt.ip.IP_PROTO_ICMP:

icmp = ip.data

self._icmp_dissect(connection, icmp)

except AttributeError:

continue

except dpkt.dpkt.NeedData:

continue

except Exception as e:

log.exception("Failed to process packet: %s", e)

file.close()

# Post processors for reconstructed flows.

self._process_smtp()

# Build results dict.

self.results["hosts"] = self.unique_hosts

self.results["domains"] = self.unique_domains

self.results["tcp"] = self.tcp_connections

self.results["udp"] = self.udp_connections

self.results["icmp"] = self.icmp_requests

self.results["http"] = self.http_requests

self.results["dns"] = self.dns_requests

self.results["smtp"] = self.smtp_requests

self.results["irc"] = self.irc_requests

"""Network analysis."""

def run(self):

self.key = "network"

results = Pcap(self.pcap_path).run()

# Save PCAP file hash.

if os.path.exists(self.pcap_path):

results["pcap_sha256"] = File(self.pcap_path).get_sha256()