I do have code like that:
# tasks.py
from __future__ import absolute_import
import celery
from bs4 import BeautifulSoup
from dateutil import parser
from sqlalchemy.orm.exc import NoResultFound
from common import scraper as scraper
from common.printer import my_print as print
class SqlAlchemyTask(celery.Task):
"""An abstract Celery Task that ensures that the connection the the
database is closed on task completion"""
abstract = True
def after_return(self, status, retval, task_id, args, kwargs, einfo):
db_session.remove()
@celery_app.task(base=SqlAlchemyTask, max_retries=10, default_retry_delay=60)
def download_task(url, date):
# some code for my task...
When I try to run celery from command line
celery --app tasks worker -l DEBUG
I got this:
Usage: celery [OPTIONS] COMMAND [ARGS]...
Error: Invalid value for '-A' / '--app':
Unable to load celery application.
While trying to load the module tasks the following error occurred:
Traceback (most recent call last):
File "/home/mint/Dev/lib/python3.6/site-packages/celery/bin/celery.py", line 53, in convert
return find_app(value)
File "/home/mint/Dev/lib/python3.6/site-packages/celery/app/utils.py", line 384, in find_app
sym = symbol_by_name(app, imp=imp)
File "/home/mint/Dev/lib/python3.6/site-packages/kombu/utils/imports.py", line 56, in symbol_by_name
module = imp(module_name, package=package, **kwargs)
File "/home/mint/Dev/lib/python3.6/site-packages/celery/utils/imports.py", line 100, in import_from_cwd
return imp(module, package=package)
File "/home/mint/Dev/lib/python3.6/importlib/__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 994, in _gcd_import
File "<frozen importlib._bootstrap>", line 971, in _find_and_load
File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 678, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/home/mint/Dev/data/downloader/tasks.py", line 8, in <module>
from common import scraper as scraper
ModuleNotFoundError: No module named 'common'
My project looks like that:
── common
│ ├── const.py
│ ├── __init__.py
│ ├── scraper.py
├── data
│ ├── downloader
│ │ ├── celery.py
│ │ ├── db.py
│ │ ├── downloaderApp.py
│ │ ├── __init__.py
│ │ └── tasks.py
├── main.py
If I run my program from main.py everything works OK. I use virtualenv. If I run downloaderApp which has no imports everything works OK too:
# downloaderApp.py
from celery import Celery
import urllib.request
import os
# Where the downloaded files will be stored
BASEDIR="/home/celery/downloadedFiles"
broker_url = "amqp://localhost"
redis_url = "redis://localhost"
app = Celery(
'downloaderApp',
broker=broker_url,
backend=redis_url)
@app.task
def download(url, filename):
"""
Download a page and save it to the BASEDIR directory
url: the url to download
filename: the filename used to save the url in BASEDIR
"""
response = urllib.request.urlopen(url)
data = response.read()
with open(BASEDIR+"/"+filename,'wb') as file:
file.write(data)
file.close()
@app.task
def add(x, y):
return x + y
I tried with relative import (If I was doing that correctly), writing imports differently, shuffling them around but nothing works. What am I missing?