forked from adamlaska/datatracker
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtextupload.py
More file actions
56 lines (42 loc) · 1.97 KB
/
textupload.py
File metadata and controls
56 lines (42 loc) · 1.97 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
# Copyright The IETF Trust 2012-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import re
from django.conf import settings
from django.core.exceptions import ValidationError
import debug # pyflakes:ignore
def get_cleaned_text_file_content(uploaded_file):
"""Read uploaded file, try to fix up encoding to UTF-8 and
transform line endings into Unix style, then return the content as
a UTF-8 string. Errors are reported as
django.core.exceptions.ValidationError exceptions."""
if not uploaded_file:
return ""
if uploaded_file.size and uploaded_file.size > 10 * 1000 * 1000:
raise ValidationError("Text file too large (size %s)." % uploaded_file.size)
content = b"".join(uploaded_file.chunks())
# try to fixup encoding
import magic
if hasattr(magic, "open"):
m = magic.open(magic.MAGIC_MIME)
m.load()
filetype = m.buffer(content)
else:
m = magic.Magic()
m.cookie = magic.magic_open(magic.MAGIC_NONE | magic.MAGIC_MIME | magic.MAGIC_MIME_ENCODING)
magic.magic_load(m.cookie, None)
filetype = m.from_buffer(content)
if not filetype.startswith(settings.DOC_TEXT_FILE_VALID_UPLOAD_MIME_TYPES):
raise ValidationError("Uploaded file does not appear to be a text file. "
"Permitted MIME types are {}, this file is {}"
.format(', '.join(settings.DOC_TEXT_FILE_VALID_UPLOAD_MIME_TYPES), filetype))
match = re.search(r"charset=([\w-]+)", filetype)
if not match:
raise ValidationError("File has unknown encoding.")
encoding = match.group(1)
try:
content = content.decode(encoding)
except Exception as e:
raise ValidationError("Error decoding file (%s). Try submitting with UTF-8 encoding or remove non-ASCII characters." % str(e))
# turn line-endings into Unix style
content = content.replace("\r\n", "\n").replace("\r", "\n")
return content