testing testing

t-vi · t-vi · commit 4a0397d53dee · 2021-02-19T17:15:01.000+01:00
diff --git a/docs/source/data_functional.rst b/docs/source/data_functional.rst
@@ -5,6 +5,10 @@ The corruption functions are modelled after
 D. Hendrycks and T. Dietterich: *Benchmarking Neural Network Robustness to Common Corruptions and Perturbations* (ICLR 2019)
 and the code provided by their authors in https://github.com/hendrycks/robustness/ but our implementation is expected to differ in details from theirs.
 
+It is important to use 0..1-values image tensors as inputs.
+This means that you need to do the normalization after applying these.
+In the traditional default PyTorch/TorchVision pipeline, images are normalized in the dataset, which may be too early. In our examples, we like to move the normalization to the beginning of the model itself or in a separate augmentation model.
+
 
 .. automodule:: torchdrift.data.functional
    :imported-members:
diff --git a/test/README.md b/test/README.md
@@ -0,0 +1,5 @@
+I run the following from the torchdrift repository:
+
+```python
+PYTHONPATH=. pytest-3  --cov-report term --cov=torchdrift test/  -v
+```
diff --git a/test/conftest.py b/test/conftest.py
@@ -0,0 +1,3 @@
+
+import os
+os.environ['NUMBA_DISABLE_JIT'] = '1'
diff --git a/test/test_corruption_functions.py b/test/test_corruption_functions.py
@@ -0,0 +1,40 @@
+import pytest
+import torchdrift
+import torch
+import scipy.ndimage
+
+# these tests could be made more specific, eventually
+
+def test_gaussian_noise():
+    a = torch.rand(4, 3, 32, 32)
+    b = torch.rand(1, 3, 200, 200)
+    a1 = torchdrift.data.functional.gaussian_noise(a)
+    b1 = torchdrift.data.functional.gaussian_noise(b)
+    assert (a1 == a1.clamp(min=0, max=1)).all()
+    assert (a-a1).std() < (b-b1).std()
+
+def test_shot_noise():
+    a = torch.rand(4, 3, 100, 100)
+    a1 = torchdrift.data.functional.shot_noise(a, severity=4)
+    a2 = torchdrift.data.functional.gaussian_noise(a, severity=4)
+    assert (a1-a).abs().max() < (a2-a).abs().max()
+    
+def test_impulse_noise():
+    a = torch.rand(4, 3, 100, 100)/2+0.25
+    a1 = torchdrift.data.functional.impulse_noise(a, severity=4)
+    assert a1.min().abs() + (1 - a1.max()).abs() < 1e-6
+
+def test_speckle_noise():
+    a = torch.rand(4, 3, 100, 100)/2+0.25
+    a1 = torchdrift.data.functional.speckle_noise(a, severity=4)
+    assert a1.min().abs() + (1 - a1.max()).abs() < 1e-6
+
+def test_gaussian_blur():
+    a = torch.rand(1, 3, 300, 300)
+    a1 = torchdrift.data.functional.gaussian_blur(a, severity=5)
+    a2 = scipy.ndimage.gaussian_filter(a, [0, 0, 6, 6])
+    assert ((a1-a2)[:,:, 32:-32, 32:-32]).max().abs() < 1e-2
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])
diff --git a/test/test_detectors.py b/test/test_detectors.py
@@ -0,0 +1,33 @@
+import pytest
+import torchdrift
+import torch
+import sklearn.decomposition
+
+def test_detector():
+    x = torch.randn(5, 5)
+    d = torchdrift.detectors.Detector()
+    d.fit(x)
+    with pytest.raises(NotImplementedError):
+        d(x)
+
+def _test_detector_class(cls):
+    torch.manual_seed(1234)
+    d = cls()
+    d2 = cls(return_p_value=True)
+    x = torch.randn(5, 5)
+    y = torch.randn(5, 5) + 1.0
+    d.fit(x)
+    d2.fit(x)
+    assert (d(x).item() < d(y).item())
+    assert d.compute_p_value(x) > 0.80
+    assert d.compute_p_value(y) < 0.05
+    assert d.compute_p_value(y) == d2(y)
+
+def test_ksdetector():
+    _test_detector_class(torchdrift.detectors.KSDriftDetector)
+
+def test_mmddetector():
+    _test_detector_class(torchdrift.detectors.KernelMMDDriftDetector)
+
+if __name__ == "__main__":
+    pytest.main([__file__])
diff --git a/test/test_utils.py b/test/test_utils.py
@@ -0,0 +1,48 @@
+import pytest
+import torchdrift
+import torch
+import torch.utils.data
+
+class TensorDataModule:
+    def __init__(self, *args):
+        self.ds = torch.utils.data.TensorDataset(*args)
+
+    def default_dataloader(self, batch_size=None, num_samples=None, shuffle=True):
+        dataset = self.ds
+        if batch_size is None:
+            batch_size = self.val_batch_size
+        replacement = num_samples is not None
+        if shuffle:
+            sampler = torch.utils.data.RandomSampler(dataset, replacement=replacement, num_samples=num_samples)
+        else:
+            sampler = None
+        return torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=sampler)
+
+def test_fit():
+    dm_ref = TensorDataModule(torch.randn(500, 5))
+    d = torchdrift.detectors.KernelMMDDriftDetector()
+    torchdrift.utils.fit(
+        dm_ref.default_dataloader(batch_size=10), torch.nn.Identity(),
+        [torch.nn.Identity(), d],
+        num_batches=3,
+        device='cpu')
+
+def test_experiment():
+    torch.manual_seed(1234)
+    d = torchdrift.detectors.KernelMMDDriftDetector()
+    dm_ref = TensorDataModule(torch.randn(500, 5))
+    dm_x = TensorDataModule(torch.randn(500, 5))
+    dm_y = TensorDataModule(torch.randn(500, 5) + 1)
+    experiment = torchdrift.utils.DriftDetectionExperiment(
+        d, torch.nn.Linear(5, 5),
+    )
+    experiment.post_training(torch.utils.data.DataLoader(dm_ref.ds, batch_size=100))
+    experiment.evaluate(dm_x, dm_y)
+    experiment = torchdrift.utils.DriftDetectionExperiment(
+        d, torch.nn.Linear(5, 5), ood_ratio=0.9, sample_size=10
+    )
+    experiment.post_training(torch.utils.data.DataLoader(dm_ref.ds, batch_size=100))
+    experiment.evaluate(dm_x, dm_y)
+
+if __name__ == "__main__":
+    pytest.main([__file__])
diff --git a/torchdrift/data/functional/corruption_functions.py b/torchdrift/data/functional/corruption_functions.py
@@ -62,7 +62,7 @@ def interpolate_severity(img, cifar, imagenet, severity):
 def gaussian_noise(x: Tensor, severity: int = 1) -> Tensor:
     """Applys gaussian noise."""
     cifar_std = [0.04, 0.06, 0.08, 0.09, 0.10]
-    imagent_std = [0.08, 0.12, 0.18, 0.26, 0.38]
+    imagenet_std = [0.08, 0.12, 0.18, 0.26, 0.38]
     std = interpolate_severity(x, cifar_std, imagenet_std, severity)
     return (x + torch.randn_like(x) * std).clamp(min=0.0, max=1.0)
 
diff --git a/torchdrift/detectors/ks.py b/torchdrift/detectors/ks.py
@@ -8,7 +8,7 @@
 try:
     import numba
     njit = numba.jit(nopython=True, fastmath=True)
-except ImportError:
+except ImportError: # pragma: no cover
     njit = lambda x: x
 
 
diff --git a/torchdrift/utils/fit.py b/torchdrift/utils/fit.py
@@ -36,7 +36,9 @@ def fit(
     nb = len(dl)
     if num_batches is not None:
         nb = min(nb, num_batches)
-    for i, (b, _) in tqdm.tqdm(zip(range(nb), dl), total=nb):
+    for i, b in tqdm.tqdm(zip(range(nb), dl), total=nb):
+        if not isinstance(b, torch.Tensor):
+            b = b[0]
         with torch.no_grad():
             all_outputs.append(feature_extractor(b.to(device)))
     all_outputs = torch.cat(all_outputs, dim=0)

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+`
	`2`	`+import os`
	`3`	`+os.environ['NUMBA_DISABLE_JIT'] = '1'`