|
|
|
@ -24,8 +24,21 @@ from sanic.log import logger |
|
|
|
|
from sanic_prometheus import monitor |
|
|
|
|
from sanic.response import stream |
|
|
|
|
from time import time |
|
|
|
|
|
|
|
|
|
_, url = sys.argv |
|
|
|
|
from urllib.parse import urlparse |
|
|
|
|
|
|
|
|
|
_, target = sys.argv |
|
|
|
|
|
|
|
|
|
# Override basic auth password from env var |
|
|
|
|
basic_auth_password = os.getenv("BASIC_AUTH_PASSWORD") |
|
|
|
|
if basic_auth_password: |
|
|
|
|
o = urlparse(target) |
|
|
|
|
netloc = o.netloc |
|
|
|
|
username = "" |
|
|
|
|
if "@" in netloc: |
|
|
|
|
username, netloc = o.netloc.split("@", 1) |
|
|
|
|
if ":" in username: |
|
|
|
|
username, _ = username.split(":") |
|
|
|
|
target = o._replace(netloc="%s:%s@%s" % (username, basic_auth_password, netloc)).geturl() |
|
|
|
|
|
|
|
|
|
AWS_ACCESS_KEY_ID = os.environ["AWS_ACCESS_KEY_ID"] |
|
|
|
|
AWS_SECRET_ACCESS_KEY = os.environ["AWS_SECRET_ACCESS_KEY"] |
|
|
|
@ -411,9 +424,9 @@ async def downloader(queue: asyncio.Queue): |
|
|
|
|
while True: |
|
|
|
|
to = aiohttp.ClientTimeout(connect=5, sock_read=2) |
|
|
|
|
async with aiohttp.ClientSession(timeout=to) as session: |
|
|
|
|
logger.info("Opening connection to %s", url) |
|
|
|
|
logger.info("Opening connection to %s", target) |
|
|
|
|
try: |
|
|
|
|
async with session.get(url) as resp: |
|
|
|
|
async with session.get(target) as resp: |
|
|
|
|
await download(resp, queue) |
|
|
|
|
except (aiohttp.ClientError, asyncio.exceptions.TimeoutError) as e: |
|
|
|
|
j = "%s.%s" % (e.__class__.__module__, e.__class__.__name__) |
|
|
|
|