Model Evaluation on Breizhcrops dataset

Dataset split:

• Train: regions FRH01, FRH02, FRH03,

• Test: region FRH04

Use only L1C for now

import os
import numpy as np
import pandas as pd
from tqdm.auto import tqdm

import sys
sys.path.append("..")
from aitlas.datasets import BreizhCropsDataset
from aitlas.utils import get_class

# setup dataset configs
test_dataset_config = {
                    "regions":["frh04"],
                    "root":"../data/breizhcrops_dataset",
                    "year":2017,
                    "level":"L1C",
                    "batch_size": 1024,
                    "shuffle": False,
                    "num_workers": 4,
                    "transforms": ["aitlas.transforms.SelectBands"]
                }
train_dataset_config = {
                    "regions":["frh01", "frh02", "frh03"],
                    "root":"../data/breizhcrops_dataset",
                    "year":2017,
                    "level":"L1C",
                    "batch_size": 1024,
                    "shuffle": False,
                    "num_workers": 4,
                    "transforms": ["aitlas.transforms.SelectBands"]
                }

Explore train dataset

train_dataset = BreizhCropsDataset(train_dataset_config)

train_dataset.show_samples()

fig1 = train_dataset.show_timeseries(100)

train_dataset.data_distribution_table()
fig2 = train_dataset.data_distribution_barchart()

train_dataset.parcel_distribution_table()

Explore test dataset

test_dataset = BreizhCropsDataset(test_dataset_config)

test_dataset.show_samples()

fig1 = test_dataset.show_timeseries(100)

test_dataset.data_distribution_table()
fig2 = test_dataset.data_distribution_barchart()

test_dataset.parcel_distribution_table()

Training the models

The models need to be trained before running this notebook, using breizhcrops_train_and_evaluate files:

python -m aitlas.run configs/breizhcrops/breizhcrops_train_and_evaluate_.json

The following models were included in this evaluation:

"OmniScaleCNN", "TempCNN","MSResNet", "InceptionTime", "LSTM","StarRNN","TransformerEncoder"

Set parameters for best models

model_configs = {
    "inceptiontime" : {
        "model": {
            "classname": "aitlas.models.InceptionTime",
            "config": {
                "input_dim":13,
                "num_classes": 9,
                "learning_rate": 0.00896,
                "weight_decay" : 0.00000222,
                "num_layers" : 3,
                "hidden_dims" : 128,
                "metrics":["accuracy","f1_score", "kappa"]
            }
        }
    },
    "lstm" : {
        "model": {
            "classname": "aitlas.models.LSTM",
            "config": {
                "input_dim":13,
                "num_classes": 9,
                "learning_rate": 0.00988,
                "weight_decay" : 0.000000526,
                "num_layers" : 4,
                "hidden_dims" : 128,
                "dropout" : 0.5713,
                "metrics":["accuracy","f1_score", "kappa"]
            }
        }
    },
    "msresnet" : {
        "model": {
            "classname": "aitlas.models.MSResNet",
            "config": {
                "input_dim":13,
                "num_classes": 9,
                "learning_rate": 0.000000627,
                "weight_decay" : 0.00000475,
                "hidden_dims" : 32,
                "metrics":["accuracy","f1_score", "kappa"]
            }
        }
    },
    "starrnn" : {
        "model": {
            "classname": "aitlas.models.StarRNN",
            "config": {
                "input_dim":13,
                "num_classes": 9,
                "learning_rate": 0.00896,
                "weight_decay" : 0.00000222,
                "num_layers" : 3,
                "hidden_dims" : 128,
                "metrics":["accuracy","f1_score", "kappa"]
            }
        }
    },
    "oscnn" : {
        "model": {
            "classname": "aitlas.models.OmniScaleCNN",
            "config": {
                "input_dim":13,
                "num_classes": 9,
                "learning_rate": 0.001066,
                "weight_decay" : 0.000000225,
                "metrics":["accuracy","f1_score", "kappa"]
            }
        }
    },
    "transformer" : {
        "model": {
            "classname": "aitlas.models.TransformerModel",
            "config": {
                "input_dim":13,
                "num_classes": 9,
                "learning_rate": 0.00131,
                "d_model" : 64,
                "dropout" : 0.4,
                "metrics":["accuracy","f1_score", "kappa"]
            }
        }
    },
    "tempcnn" : {
        "model": {
            "classname": "aitlas.models.TempCNN",
            "config": {
                "input_dim":13,
                "num_classes": 9,
                "sequence_length": 45,
                "learning_rate": 0.000238,
                "weight_decay" : 0.0000518,
                "kernel_size" : 7,
                "hidden_dims" : 128,
                "dropout" : 0.18,
                "metrics":["accuracy","f1_score", "kappa"]
            }
        }
    }
}

Experiment Folder structure

examples/
    experiment/
        breizhcrops/
            <model>/
                checkpoint_<ts>_.pth.tar
            ...

def load_model(run):
    """
    Load latest checkpoint for model
    """
    models = [l for l in os.listdir(os.path.join(logdir,run)) if l.startswith("checkpoint")]
    last_model = models[0]
    for model in models:
        timestamp = int(model.split('.')[0].split("_")[1])
        if timestamp > int(last_model.split('.')[0].split("_")[1]):
            last_model=model
    model_path = os.path.join(logdir,run,last_model)
    model_cls = get_class(model_configs[run]['model']['classname'])
    model = model_cls(model_configs[run]['model']['config'])
    model.prepare()
    model.load_model(model_path)

    return model

Compile L1C Results

# specify where the "root" folder of the models for the dataset
logdir = "examples/experiment/breizhcrops"

for model_key in model_configs.keys():
    m = load_model(model_key) # load model
    result = m.evaluate_model(test_dataset.dataloader()) # run predictions on test dataset

    # print metrics for model
    print(f"Model: {m.__class__.__name__}")
    print("")
    for metric in [m.running_metrics.accuracy(), m.running_metrics.precision(), m.running_metrics.recall(), m.running_metrics.f1_score()]:
        for k in metric.keys():
            print(f"{k} {metric[k]}")
        print("")