hi, I am going through lesson 1 and noticed that the download_url progress bar is showing incorrect values when downloading the birds and forest images from DuckDuckGo.

Hi fastai folk,

I noticed the LICENSE file for the project is Apache-2.0, but the setup.py lists 5 different licenses. Is the setup.py in error? It feels like maybe it's a template and still needs work?

I was using simple urllib request GET method to download files, now I want to shift to fastdownload. The url was getting downloaded before without SSL check. I want to bypass this SSL check. How can i do this

Error:
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/fastdownload/core.py", line 117, in get self.download(url, force=force) File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/fastdownload/core.py", line 92, in download return download_and_check(url, urldest(url, self.arch_path()), self.module, force) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/fastdownload/core.py", line 61, in download_and_check res = download_url(url, fpath) ^^^^^^^^^^^^^^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/fastdownload/core.py", line 19, in download_url return urlsave(url, dest, reporthook=progress if show_progress else None, timeout=timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/fastcore/net.py", line 184, in urlsave nm,msg = urlretrieve(url, dest, reporthook, headers=headers, timeout=timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/fastcore/net.py", line 149, in urlretrieve with contextlib.closing(urlopen(url, data, headers=headers, timeout=timeout)) as fp: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/fastcore/net.py", line 108, in urlopen try: return urlopener().open(urlwrap(url, data=data, headers=headers), timeout=timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/urllib/request.py", line 519, in open response = self._open(req, data) ^^^^^^^^^^^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/urllib/request.py", line 536, in _open result = self._call_chain(self.handle_open, protocol, protocol + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/urllib/request.py", line 496, in _call_chain result = func(*args) ^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/urllib/request.py", line 1391, in https_open return self.do_open(http.client.HTTPSConnection, req, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/urllib/request.py", line 1351, in do_open raise URLError(err) urllib.error.URLError: <urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:992)>

checks_module function returns fmod as type Path or Dictionary
so one solution can be to replace if not fmod.exists(): return {} with:

if not type(fmod)==PosixPath: return {}
or
try: fmod.exists()

or something better

I was trying to use fastdownload (I am using fastdownload-0.0.5) for downloading some compressed files and I was testing force=True argument. Code looked something like this:

import fastdownload
from fastdownload import FastDownload

d = FastDownload(module=fastdownload)
path = d.get('https://silkdb.bioinfotoolkits.net/__resource/Bombyx_mori/download/cds.fa.tar.gz')
path = d.get('https://silkdb.bioinfotoolkits.net/__resource/Bombyx_mori/download/cds.fa.tar.gz', force=True)

---------------------------------------------------------------------------
NotADirectoryError                        Traceback (most recent call last)
fastdownload.ipynb Cell 12' in <cell line: 1>()
----> [1] path = d.get('https://silkdb.bioinfotoolkits.net/__resource/Bombyx_mori/download/cds.fa.tar.gz', force=True)
      [2] path

File ~/my-conda-envs/tenv/lib/python3.8/site-packages/fastdownload/core.py:122, in FastDownload.get(self, url, extract_key, force)
    [120]     if data.exists(): return data
    [121] self.download(url, force=force)
--> [122] return self.extract(url, extract_key=extract_key, force=force)

File ~/my-conda-envs/tenv/lib/python3.8/site-packages/fastdownload/core.py:114, in FastDownload.extract(self, url, extract_key, force)
    [112] dest = self.data_path(extract_key)
    [113] dest.mkdir(exist_ok=True, parents=True)
--> [114] return untar_dir(arch, dest, rename=True, overwrite=force)

File ~/my-conda-envs/tenv/lib/python3.8/site-packages/fastcore/xtras.py:226, in untar_dir(fname, dest, rename, overwrite)
    [224]     dest = dest/src.name
    [225] if dest.exists():
--> [226]     if overwrite: shutil.rmtree(dest)
    [227]     else: return dest
    [228] if rename: src = _unpack(fname, out)

File ~/my-conda-envs/tenv/lib/python3.8/shutil.py:718, in rmtree(path, ignore_errors, onerror)
    [716] try:
    [717]     if os.path.samestat(orig_st, os.fstat(fd)):
--> [718]         _rmtree_safe_fd(fd, path, onerror)
    [719]         try:
    [720]            os.rmdir(path)

File ~/my-conda-envs/tenv/lib/python3.8/shutil.py:631, in _rmtree_safe_fd(topfd, path, onerror)
    [629] except OSError as err:
    [630]     err.filename = path
--> [631]     onerror(os.scandir, path, sys.exc_info())
    [632]     return
    [633] for entry in entries:

File ~/my-conda-envs/tenv/lib/python3.8/shutil.py:627, in _rmtree_safe_fd(topfd, path, onerror)
    [625] def _rmtree_safe_fd(topfd, path, onerror):
    [626]     try:
--> [627]         with os.scandir(topfd) as scandir_it:
    [628]             entries = list(scandir_it)
    [629]     except OSError as err:

NotADirectoryError: [Errno 20] Not a directory: Path('/home/jovyan/.fastdownload/data/cds.fa')

What is the reason behind this? Is there a way to get around this?

When I want to use fastdownload for other urls's I get an unknown archive format error.

Here is the code

from fastdownload import FastDownload

d = FastDownload()

d.get('http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz')

# Convert to zip file if gzip
        file_type = magic.from_file(arch)
        if 'gzip' in file_type:
            with gzip.open(arch, 'rb') as f_in:
                file_content = f_in.read()
                arch.write_bytes(file_content)

            with zipfile.ZipFile(arch, 'w') as myzip:
                myzip.write(arch)
                # rename file
                new_arch = arch.with_suffix('.zip')
                arch = arch.rename(new_arch)