How to upload file to the server with unicode content?

Question

I try to use the following code to upload files to the server:

def build_request(self, theurl, fields, files, txheaders=None):
    content_type, body = self.encode_multipart_formdata(fields, files)
    if not txheaders: txheaders = {}
    txheaders['Content-type'] = content_type
    txheaders['Content-length'] = str(len(body))
    return urllib2.Request(theurl, body, txheaders)

def encode_multipart_formdata(self,fields, files, BOUNDARY = '-----'+mimetools.choose_boundary()+'-----'):
    ''' from www.voidspace.org.uk/atlantibots/pythonutils.html '''
    CRLF = '\r\n'
    L = []
    if isinstance(fields, dict):
        fields = fields.items()
    for (key, value) in fields:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"' % key)
        L.append('')
        L.append(value)
    for (key, filename, value) in files:
        filetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
        L.append('Content-Type: %s' % filetype)
        L.append('')
        L.append(value)
    L.append('--' + BOUNDARY + '--')
    L.append('')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    return content_type, body

mp3 = ('file', file, open(file,'rb').read())
url = self.build_request(upload_url, {}, (mp3,))
res = urllib2.urlopen(url)

I upload mp3 file (0001.mp3) and get the following error -

(type 'exceptions.UnicodeDecodeError', UnicodeDecodeError('ascii', '-------192.1xx.xx.xx.501.5413.1420317280.341.1-----\r\nContent-Disposition: form-data; name="file"; filename="/Users/Me/Music/0001.mp3"\r\nContent-Type: audio/mpeg\r\n\r\nID3\x04\x00\x00\x00\x00\x00#TSSE\x00\x00\x00\x0f\x00\x00\x03Lavf55.33.100\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfb\x94\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Info\x00\x00\x00\x07\x00\x00#\xf9\x005\xf7\x00\x00\x02\x05\x08\n\r\x10\x12\x14\x17\x19\x1c\x1f!$\'(+.0368;=@BEHJMOQTWY\_acfhknpsvxz}\x80\x82\x85\x88\x89\x8c\x8f\x91\x94\x97\x99\x9c\x9e\xa1\xa3\xa6\xa9\xab\xae\xb1......UUUUU', 45, 46, 'ordinal not in range(128)'), )

What is wrong there?

Upd. The full traceback is below:

Traceback (most recent call last):
  File "test.py", line 258, in <module>
    upload()
  File "test.py", line 242, in upload
    success = uploadFile(file)
  File "test.py", line 179, in uploadFile
    res = urllib2.urlopen(url)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 127, in urlopen
    return _opener.open(url, data, timeout)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 404, in open
    response = self._open(req, data)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 422, in _open
    '_open', req)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 382, in _call_chain
    result = func(*args)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1214, in http_open
    return self.do_open(httplib.HTTPConnection, req)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1181, in do_open
    h.request(req.get_method(), req.get_selector(), req.data, headers)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 973, in request
    self._send_request(method, url, body, headers)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 1007, in _send_request
    self.endheaders(body)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 969, in endheaders
    self._send_output(message_body)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 827, in _send_output
    msg += message_body
UnicodeDecodeError: 'ascii' codec can't decode byte 0xff in position 220: ordinal not in range(128)

@MartijnPieters, I've updated the question with the traceback. — LA_
You have a unicode header. What headers are you sending? That could include the URL. — Martijn Pieters♦
@MartijnPieters, I don't add anything to the headers manually, i.e. the headers are generated in accordance with the code above. — LA_
@MartijnPieters, you were right - the problem was with Unicode URL used, please post it as the answer, so I'll delete mine and will accept yours. — LA_

Martijn Pieters Martijn Pieters · Accepted Answer · 2015-01-04T11:13:47

The exception traceback shows that Python tried to decode your request body; since it is binary data the default encoding used by Python for implicit decoding (ASCII) fails here.

Python tried to decode your request body because the first part to be sent to the server, the HTTP headers (including the initial request line with method, path and HTTP version), is already a unicode object. This happens if at least one of your headers or your URL is a unicode object and the remainder was decodable as ASCII.

Make sure your headers and URL are encoded to bytes. If they are not ASCII you'll need to explicitly encode them; headers generally use Latin1 or opaque bytes (HTTP 1.0 offered a mime-compatible encoding options that no one ever uses), the URL must be ASCII with any path elements or query parameters encoded to UTF-8 then URL encoded (see how to deal with ® in url for urllib2.urlopen? for sample code).

Since you don't pass in extra headers, I'll assume it is your URL that is a unicode object here:

url = self.build_request(upload_url.encode('ascii'), {}, (mp3,))

How to upload file to the server with unicode content?

1 Answers