I made this ugly fix by overriding some org mode elisp. I put this in my .emacs and now things work as expected. Maybe I will post a patch to org mode when I get the time.
(defun fd--org-doc-begin ()
"Skip all the initial export options"
(save-excursion
(goto-char (point-min))
(while (and (or
(looking-at "[ \t]*#+")
(looking-at "[ \t]*$"))
(progn (next-line) (< (point) (point-max))))
(beginning-of-line))
(point)))
;;; This was overriden from ox.el
(defun org-export--prepare-file-contents
(file &optional lines ind minlevel id footnotes with-export-options)
"Prepare contents of FILE for inclusion and return it as a string.
When optional argument LINES is a string specifying a range of
lines, include only those lines.
Optional argument IND, when non-nil, is an integer specifying the
global indentation of returned contents. Since its purpose is to
allow an included file to stay in the same environment it was
created (e.g., a list item), it doesn't apply past the first
headline encountered.
Optional argument MINLEVEL, when non-nil, is an integer
specifying the level that any top-level headline in the included
file should have.
Optional argument ID is an integer that will be inserted before
each footnote definition and reference if FILE is an Org file.
This is useful to avoid conflicts when more than one Org file
with footnotes is included in a document.
Optional argument FOOTNOTES is a hash-table to store footnotes in
the included document.
Optional argument WITH-EXPORT-OPTIONS will stop this function
from ignoring export options at the beginning of the file."
(with-temp-buffer
(insert-file-contents file)
(when (not with-export-options)
(narrow-to-region (fd--org-doc-begin) (point-max)))
(when lines
(let* ((lines (split-string lines "-"))
(lbeg (string-to-number (car lines)))
(lend (string-to-number (cadr lines)))
(beg (if (zerop lbeg) (point-min)
(goto-char (point-min))
(forward-line (1- lbeg))
(point)))
(end (if (zerop lend) (point-max)
(goto-char (point-min))
(forward-line (1- lend))
(point))))
(narrow-to-region beg end)))
;; Remove blank lines at beginning and end of contents. The logic
;; behind that removal is that blank lines around include keyword
;; override blank lines in included file.
(goto-char (point-min))
(org-skip-whitespace)
(beginning-of-line)
(delete-region (point-min) (point))
(goto-char (point-max))
(skip-chars-backward " \r\t\n")
(forward-line)
(delete-region (point) (point-max))
;; If IND is set, preserve indentation of include keyword until
;; the first headline encountered.
(when (and ind (> ind 0))
(unless (eq major-mode 'org-mode)
(let ((org-inhibit-startup t)) (org-mode)))
(goto-char (point-min))
(let ((ind-str (make-string ind ?\s)))
(while (not (or (eobp) (looking-at org-outline-regexp-bol)))
;; Do not move footnote definitions out of column 0.
(unless (and (looking-at org-footnote-definition-re)
(eq (org-element-type (org-element-at-point))
'footnote-definition))
(insert ind-str))
(forward-line))))
;; When MINLEVEL is specified, compute minimal level for headlines
;; in the file (CUR-MIN), and remove stars to each headline so
;; that headlines with minimal level have a level of MINLEVEL.
(when minlevel
(unless (eq major-mode 'org-mode)
(let ((org-inhibit-startup t)) (org-mode)))
(org-with-limited-levels
(let ((levels (org-map-entries
(lambda () (org-reduced-level (org-current-level))))))
(when levels
(let ((offset (- minlevel (apply #'min levels))))
(unless (zerop offset)
(when org-odd-levels-only (setq offset (* offset 2)))
;; Only change stars, don't bother moving whole
;; sections.
(org-map-entries
(lambda ()
(if (< offset 0) (delete-char (abs offset))
(insert (make-string offset ?*)))))))))))
;; Append ID to all footnote references and definitions, so they
;; become file specific and cannot collide with footnotes in other
;; included files. Further, collect relevant footnote definitions
;; outside of LINES, in order to reintroduce them later.
(when id
(let ((marker-min (point-min-marker))
(marker-max (point-max-marker))
(get-new-label
(lambda (label)
;; Generate new label from LABEL by prefixing it with
;; "-ID-".
(format "-%d-%s" id label)))
(set-new-label
(lambda (f old new)
;; Replace OLD label with NEW in footnote F.
(save-excursion
(goto-char (+ (org-element-property :begin f) 4))
(looking-at (regexp-quote old))
(replace-match new))))
(seen-alist))
(goto-char (point-min))
(while (re-search-forward org-footnote-re nil t)
(let ((footnote (save-excursion
(backward-char)
(org-element-context))))
(when (memq (org-element-type footnote)
'(footnote-definition footnote-reference))
(let* ((label (org-element-property :label footnote)))
;; Update the footnote-reference at point and collect
;; the new label, which is only used for footnotes
;; outsides LINES.
(when label
(let ((seen (cdr (assoc label seen-alist))))
(if seen (funcall set-new-label footnote label seen)
(let ((new (funcall get-new-label label)))
(push (cons label new) seen-alist)
(org-with-wide-buffer
(let* ((def (org-footnote-get-definition label))
(beg (nth 1 def)))
(when (and def
(or (< beg marker-min)
(>= beg marker-max)))
;; Store since footnote-definition is
;; outside of LINES.
(puthash new
(org-element-normalize-string (nth 3 def))
footnotes))))
(funcall set-new-label footnote label new)))))))))
(set-marker marker-min nil)
(set-marker marker-max nil)))
(org-element-normalize-string (buffer-string))))
