General template for create a new dataset from scratch¶

%load_ext autoreload
%autoreload 2

# Basic utility functions
import logging
import os
import pathlib
from pprint import pprint

from src.log import logger
from src import paths
from src.utils import list_dir
from functools import partial

# data functions
from src.data import DataSource, Dataset, DatasetGraph, Catalog
from src import helpers

# Optionally set to debug log level
logger.setLevel(logging.DEBUG)

ds_name = 'covid-19-epidemiology-raw'
dsrc = DataSource(ds_name)

url = 'https://storage.googleapis.com/covid19-open-data/v2/epidemiology.csv'

filename = 'epidemiology.csv' # path relative to paths['raw_data_path'] for the file

license = """
[CC-BY 4.0](https://github.com/GoogleCloudPlatform/covid-19-open-data/blob/main/output/CC-BY)
"""

metadata = """
The epidemiology table from Google's [COVID-19 Open-Data dataset](https://github.com/GoogleCloudPlatform/covid-19-open-data). 

The full dataset contains datasets of daily time-series data related to COVID-19 for over 20,000 distinct locations around the world. The data is at the spatial resolution of states/provinces for most regions and at county/municipality resolution for many countries such as Argentina, Brazil, Chile, Colombia, Czech Republic, Mexico, Netherlands, Peru, United Kingdom, and USA. All regions are assigned a unique location key, which resolves discrepancies between ISO / NUTS / FIPS codes, etc. The different aggregation levels are:

    0: Country
    1: Province, state, or local equivalent
    2: Municipality, county, or local equivalent
    3: Locality which may not follow strict hierarchical order, such as "city" or "nursing homes in X location"

There are multiple types of data:

    Outcome data Y(i,t), such as cases, tests, hospitalizations, deaths and recoveries, for region i and time t
    Static covariate data X(i), such as population size, health statistics, economic indicators, geographic boundaries
    Dynamic covariate data X(i,t), such as mobility, search trends, weather, and government interventions

The data is drawn from multiple sources, as listed below, and stored in separate tables as CSV files grouped by context, which can be easily merged due to the use of consistent geographic (and temporal) keys as it is done for the main table.

One of these files is the epidemiology.csv file used here.
"""

dsrc.add_url(url=url, file_name=filename, unpack_action='copy')
dsrc.add_metadata(contents=metadata, force=True)
dsrc.add_metadata(contents=license, kind='LICENSE', force=True)

dsrc.file_dict

from src.data.fileset import process_fileset_files
process_function = process_fileset_files
process_function_kwargs = {'file_glob':'*.csv',
                           'do_copy': True,
                           'fileset_dir': ds_name+'.fileset',
                           'extract_dir': ds_name}

help(process_function)

dsrc.process_function = partial(process_function, **process_function_kwargs)

dsrc.update_catalog()

dsc = Catalog.load('datasources')
dsc[ds_name]

%%time
dsrc.fetch()

%%time
dsrc.unpack()

from src.data import DatasetGraph

paths['catalog_path']

dag = DatasetGraph(catalog_path=paths['catalog_path'])

dag.sources

dsc = Catalog.load('datasources'); dsc

dag.add_source(output_dataset=ds_name, datasource_name=ds_name, overwrite_catalog=True)

dc = Catalog.load('datasets'); dc

%%time
ds = Dataset.from_catalog(ds_name)

%%time
ds = Dataset.load(ds_name)

pprint(ds.metadata)

print(ds.LICENSE)

ds.FILESET

ds.fileset_file('epidemiology.csv')

ds.data is None

ds.target is None