dmlc
diff --git a/‎Jenkinsfile
Lines changed: 29 additions & 0 deletions b/‎Jenkinsfile
Lines changed: 29 additions & 0 deletions
diff --git a/‎docs/build.yml
Lines changed: 1 addition & 0 deletions b/‎docs/build.yml
Lines changed: 1 addition & 0 deletions
diff --git a/‎docs/conf.py
Lines changed: 1 addition & 1 deletion b/‎docs/conf.py
Lines changed: 1 addition & 1 deletion
diff --git a/‎docs/tutorials/auto/demo_auto_detection.py
Lines changed: 64 additions & 0 deletions b/‎docs/tutorials/auto/demo_auto_detection.py
Lines changed: 64 additions & 0 deletions
diff --git a/‎docs/tutorials/detection/finetune_detection.py
Lines changed: 1 addition & 1 deletion b/‎docs/tutorials/detection/finetune_detection.py
Lines changed: 1 addition & 1 deletion
diff --git a/‎docs/tutorials/detection/train_yolo_v3.py
Lines changed: 5 additions & 5 deletions b/‎docs/tutorials/detection/train_yolo_v3.py
Lines changed: 5 additions & 5 deletions
diff --git a/‎docs/tutorials/pose/cam_demo.py
Lines changed: 4 additions & 4 deletions b/‎docs/tutorials/pose/cam_demo.py
Lines changed: 4 additions & 4 deletions
diff --git a/‎gluoncv/auto/__init__.py
Lines changed: 2 additions & 0 deletions b/‎gluoncv/auto/__init__.py
Lines changed: 2 additions & 0 deletions
diff --git a/‎gluoncv/auto/data/__init__.py
Lines changed: 2 additions & 0 deletions b/‎gluoncv/auto/data/__init__.py
Lines changed: 2 additions & 0 deletions
diff --git a/‎gluoncv/auto/data/auto_data.py
Lines changed: 201 additions & 0 deletions b/‎gluoncv/auto/data/auto_data.py
Lines changed: 201 additions & 0 deletions
@@ -85,6 +85,35 @@ stage("Unit Test") {
         }
       }
     }
+  },
+  'Auto': {
+    node('linux-gpu') {
+      ws('workspace/gluon-cv-py3-auto') {
+        timeout(time: max_time, unit: 'MINUTES') {
+          checkout scm
+          VISIBLE_GPU=env.EXECUTOR_NUMBER.toInteger() % 8
+          sh """#!/bin/bash
+          conda env remove -n gluon-cv-py3-auto_test -y
+          set -ex
+          # remove and create new env instead
+          conda env create -n gluon-cv-py3-auto_test -f tests/py3_auto.yml
+          conda env update -n gluon-cv-py3-auto_test -f tests/py3_auto.yml --prune
+          conda activate gluon-cv-py3-auto_test
+          conda list
+          export CUDA_VISIBLE_DEVICES=${VISIBLE_GPU}
+          export KMP_DUPLICATE_LIB_OK=TRUE
+          make clean
+          # from https://stackoverflow.com/questions/19548957/can-i-force-pip-to-reinstall-the-current-version
+          pip install --upgrade --force-reinstall --no-deps .
+          env
+          export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64
+          export MPLBACKEND=Agg
+          export MXNET_CUDNN_AUTOTUNE_DEFAULT=0
+          nosetests --with-timer --timer-ok 60 --timer-warning 120 -x --with-coverage --cover-package gluoncv -v tests/auto
+          """
+        }
+      }
+    }
   }
 }
 
 
@@ -23,3 +23,4 @@ dependencies:
   - decord
   - cython
   - pycocotools
+  - autocfg
@@ -87,7 +87,7 @@
         'build/examples_distributed',
         'build/examples_deployment',],
 
-    'filename_pattern': '.py',
+    'filename_pattern': '.pydisabled',
     'ignore_pattern': 'im2rec.py',
     'expected_failing_examples': [],
 
 
@@ -0,0 +1,64 @@
+"""01. HPO with Faster R-CNN end-to-end on PASCAL VOC
+=====================================================
+
+This tutorial goes through the basic steps of using AutoGluon to tune hyper-parameters for a
+Faster-RCNN [Ren15]_ object detection model provided by GluonCV.
+"""
+
+##########################################################
+# Dataset
+# -------
+#
+# Please first go through this :ref:`sphx_glr_build_examples_datasets_pascal_voc.py` tutorial to setup Pascal
+# VOC dataset on your disk.
+# Then, we are ready to load training and validation images.
+
+import autogluon as ag
+
+from gluoncv.auto.estimators.faster_rcnn import FasterRCNNEstimator
+from gluoncv.auto.tasks.object_detection import ObjectDetection
+
+# Define search space
+time_limits = 60 * 60  # 1hr
+search_args = {'dataset': 'voc', 'split_ratio': 0.8, 'num_trials': 30,
+               'epochs': ag.Categorical(30, 40, 50, 60), 'num_workers': 16,
+               'net': ag.Categorical('resnest101', 'resnest50'), 'meta_arch': 'faster_rcnn',
+               'search_strategy': 'random', 'search_options': {},
+               'lr': ag.Categorical(0.005, 0.002, 2e-4, 5e-4), 'transfer': False,
+               'data_shape': (640, 800), 'nthreads_per_trial': 12, 'verbose': False,
+               'ngpus_per_trial': 4, 'batch_size': 4, 'hybridize': True,
+               'lr_decay_epoch': ag.Categorical([24, 28], [35], [50, 55], [40], [45], [55],
+                                                [30, 35], [20]),
+               'warmup_iters': ag.Int(5, 500), 'resume': False, 'checkpoint': 'checkpoint/exp1.ag',
+               'visualizer': 'none', 'start_epoch': 0, 'lr_mode': 'step', 'lr_decay': 0.1,
+               'lr_decay_period': 0, 'warmup_lr': 0.0, 'warmup_epochs': 2, 'warmup_factor': 1. / 3.,
+               'momentum': 0.9, 'log_interval': 100, 'save_prefix': '', 'save_interval': 10,
+               'val_interval': 1, 'num_samples': -1, 'no_random_shape': False, 'no_wd': False,
+               'mixup': False, 'no_mixup_epochs': 20, 'reuse_pred_weights': True, 'horovod': False,
+               'grace_period': None, 'auto_search': True, 'seed': 223,
+               'wd': ag.Categorical(1e-4, 5e-4, 2.5e-4), 'syncbn': ag.Bool(), 'label_smooth': False,
+               'time_limits': time_limits, 'dist_ip_addrs': []}
+
+# Construct a object detection task based on the config.
+task = ObjectDetection(search_args, FasterRCNNEstimator)
+
+# Automatically fit a model.
+estimator = task.fit()
+
+# Evaluate the final model on test set.
+test_map = estimator.evaluate()
+print("mAP on test dataset: {}".format(test_map[-1][-1]))
+print(test_map)
+
+# Save our final model.
+estimator.save('final_model.model')
+
+##########################################################
+# References
+# ----------
+#
+# .. [Girshick14] Ross Girshick and Jeff Donahue and Trevor Darrell and Jitendra Malik. Rich Feature Hierarchies for Accurate Object Detection and Semantic Segmentation. CVPR 2014.
+# .. [Girshick15] Ross Girshick. Fast {R-CNN}. ICCV 2015.
+# .. [Ren15] Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun. Faster {R-CNN}: Towards Real-Time Object Detection with Region Proposal Networks. NIPS 2015.
+# .. [He16] Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun. Deep Residual Learning for Image Recognition. CVPR 2016.
+# .. [Lin17] Tsung-Yi Lin and Piotr Dollár and Ross Girshick and Kaiming He and Bharath Hariharan and Serge Belongie. Feature Pyramid Networks for Object Detection. CVPR 2017.
@@ -170,4 +170,4 @@ def get_dataloader(net, train_dataset, data_shape, batch_size, num_workers):
 #     adapting a training blocks in the following examples:
 #
 #     :download:`Download train_faster_rcnn.py<../../../scripts/detection/faster_rcnn/train_faster_rcnn.py>`
-#     :download:`Download train_yolo3.py<../../../scripts/detection/yolo/train_yolo3.py>`
+#     :download:`Download train_yolo.py<../../../scripts/detection/yolo/train_yolo.py>`
@@ -12,7 +12,7 @@
     You can skip the rest of this tutorial and start training your YOLOv3 model
     right away by downloading this script:
 
-    :download:`Download train_yolo3.py<../../../scripts/detection/yolo/train_yolo3.py>`
+    :download:`Download train_yolo.py<../../../scripts/detection/yolo/train_yolo.py>`
     Random shape training requires more GPU memory but generates better results. You can turn it off by setting `--no-random-shape`.
 
     Example usage:
@@ -21,19 +21,19 @@
 
     .. code-block:: bash
 
-        python train_yolo3.py --gpus 0
+        python train_yolo.py --gpus 0
 
     Train a darknet53 model on GPU 0,1,2,3 with synchronize BatchNorm:
 
     .. code-block:: bash
 
-        python train_yolo3.py --gpus 0,1,2,3 --network darknet53 --syncbn
+        python train_yolo.py --gpus 0,1,2,3 --network darknet53 --syncbn
 
     Check the supported arguments:
 
     .. code-block:: bash
 
-        python train_yolo3.py --help
+        python train_yolo.py --help
 
 
 .. hint::
@@ -234,7 +234,7 @@
 #
 # .. hint::
 #
-#   Please checkout the full :download:`training script <../../../scripts/detection/yolo/train_yolo3.py>` for complete implementation.
+#   Please checkout the full :download:`training script <../../../scripts/detection/yolo/train_yolo.py>` for complete implementation.
 
 
 ##########################################################
 
@@ -52,7 +52,7 @@
     detector.reset_class(classes=['person'], reuse_weights={'person':'person'})
     detector.hybridize()
 
-Next for the estimator, we choose ``simple_pose_resnet18_v1b`` for it is light-weighted.
+Next for the estimators, we choose ``simple_pose_resnet18_v1b`` for it is light-weighted.
 
 The default ``simple_pose_resnet18_v1b`` model was trained with input size 256x192.
 We also provide an optional ``simple_pose_resnet18_v1b`` model trained with input size 128x96.
@@ -61,8 +61,8 @@
 
 .. code-block:: python
 
-    estimator = get_model('simple_pose_resnet18_v1b', pretrained='ccd24037', ctx=ctx)
-    estimator.hybridize()
+    estimators = get_model('simple_pose_resnet18_v1b', pretrained='ccd24037', ctx=ctx)
+    estimators.hybridize()
 
 With OpenCV, we can easily retrieve frames from the webcam.
 
@@ -107,7 +107,7 @@
         pose_input, upscale_bbox = detector_to_simple_pose(frame, class_IDs, scores, bounding_boxs,
                                                            output_shape=(128, 96), ctx=ctx)
         if len(upscale_bbox) > 0:
-            predicted_heatmap = estimator(pose_input)
+            predicted_heatmap = estimators(pose_input)
             pred_coords, confidence = heatmap_to_coord(predicted_heatmap, upscale_bbox)
 
             img = cv_plot_keypoints(frame, pred_coords, confidence, class_IDs, bounding_boxs, scores,
 
@@ -0,0 +1,2 @@
+"""GluonCV auto"""
+from .estimators import *
@@ -0,0 +1,2 @@
+"""Data Pipelines"""
+from .auto_data import url_data, URLs, is_url
@@ -0,0 +1,201 @@
+"""Auto data preparation."""
+import os
+import re
+from pathlib import Path
+import yaml
+from ...utils.download import download
+from ...utils.filesystem import unzip, untar, PathTree
+
+def url_data(url, path=None, overwrite=False, overwrite_folder=False, sha1_hash=None, root=None, disp_depth=1):
+    """Download an given URL
+
+    Parameters
+    ----------
+    url : str
+        URL to download
+    path : str, optional
+        Destination path to store downloaded file. By default stores to the
+        ~/.gluoncv directory with same name as in url.
+        You can also change the default behavior by editing ~/.gluoncv/config.yaml.
+    overwrite : bool, optional
+        Whether to overwrite destination file if already exists.
+    overwrite_folder : bool, optional
+        Whether to extract file to destination folder if already exists. You may use this option if you suspect
+        the destination is corrupted or some files are missing.
+    sha1_hash : str, optional
+        Expected sha1 hash in hexadecimal digits. Will ignore existing file when hash is specified
+        but doesn't match.
+    root : str, optional
+        Destination root dir to store extracted files. By default it's default in ~/.gluoncv directory.
+    disp_depth : int, optional
+        If set > 0, will print out the tree structure of extracted dataset folder with maximum `disp_depth`.
+
+    Returns
+    -------
+    str or tuple of str
+        The file path of the downloaded file.
+    """
+    fname = Path(path or URLs.path(url, c_key='archive'))
+    fname.parent.mkdir(parents=True, exist_ok=True)
+    fname = download(url, path=str(fname.resolve()), overwrite=overwrite, sha1_hash=sha1_hash)
+    extract_root = URLs.path(url, c_key='data')
+    extract_root = extract_root.parent.joinpath(extract_root.stem)
+    extract_root.mkdir(parents=True, exist_ok=True)
+    if fname.endswith('.zip'):
+        folder = unzip(fname, root=root if root else extract_root, strict=overwrite_folder)
+    elif fname.endswith('gz'):
+        folder = untar(fname, root=root if root else extract_root, strict=overwrite_folder)
+    else:
+        raise ValueError('Unknown url data with file: {}'.format(fname))
+
+    if disp_depth > 0:
+        path_tree = PathTree(folder, disp_depth)
+        print(path_tree)
+
+    return Path(folder)
+
+
+class Config:
+    "Setup config at `~/.gluoncv` unless it exists already."
+    config_path = Path(os.getenv('MXNET_HOME', '~/.gluoncv')).expanduser()
+    config_file = config_path/'config.yml'
+
+    def __init__(self):
+        self.config_path.mkdir(parents=True, exist_ok=True)
+        if not self.config_file.exists():
+            self.create_config()
+        self.d = self.load_config()
+
+    def __getitem__(self, k):
+        k = k.lower()
+        if k not in self.d:
+            k = k + '_path'
+        return Path(self.d[k])
+
+    def __getattr__(self, k):
+        if k == 'd':
+            raise AttributeError
+        return self[k]
+
+    def __setitem__(self, k, v):
+        self.d[k] = str(v)
+    def __contains__(self, k):
+        return k in self.d
+
+    def load_config(self):
+        "load and return config if version equals 2 in existing, else create new config."
+        with open(self.config_file, 'r') as f:
+            config = yaml.safe_load(f)
+            if 'version' in config and config['version'] == 2:
+                return config
+            elif 'version' in config:
+                self.create_config(config)
+            else:
+                self.create_config()
+        return self.load_config()
+
+    def create_config(self, cfg=None):
+        "create new config with default paths and set `version` to 2."
+        config = {'data_path':    str(self.config_path/'datasets'),
+                  'archive_path': str(self.config_path/'archive'),
+                  'storage_path': '/tmp',
+                  'model_path':   str(self.config_path/'models'),
+                  'version':      2}
+        if cfg is not None:
+            cfg['version'] = 2
+            config = merge(config, cfg)
+        self.save_file(config)
+
+    def save(self):
+        self.save_file(self.d)
+
+    def save_file(self, config):
+        "save config file at default config location `~/.gluoncv/config.yml`."
+        with self.config_file.open('w') as f:
+            yaml.dump(config, f, default_flow_style=False)
+
+
+# pylint: disable=bad-whitespace
+class URLs():
+    "Global constants for dataset and model URLs."
+    LOCAL_PATH = Path.cwd()
+    MDL = 'http://files.fast.ai/models/'
+    S3  = 'https://s3.amazonaws.com/fast-ai-'
+    URL = f'{S3}sample/'
+
+    S3_IMAGE    = f'{S3}imageclas/'
+    S3_IMAGELOC = f'{S3}imagelocal/'
+    S3_COCO     = f'{S3}coco/'
+
+    # main datasets
+    ADULT_SAMPLE        = f'{URL}adult_sample.tgz'
+    BIWI_SAMPLE         = f'{URL}biwi_sample.tgz'
+    CIFAR               = f'{URL}cifar10.tgz'
+    COCO_SAMPLE         = f'{S3_COCO}coco_sample.tgz'
+    COCO_TINY           = f'{S3_COCO}coco_tiny.tgz'
+    HUMAN_NUMBERS       = f'{URL}human_numbers.tgz'
+    # IMDB                = f'{S3_NLP}imdb.tgz'
+    IMDB_SAMPLE         = f'{URL}imdb_sample.tgz'
+    ML_SAMPLE           = f'{URL}movie_lens_sample.tgz'
+    ML_100k             = 'http://files.grouplens.org/datasets/movielens/ml-100k.zip'
+    MNIST_SAMPLE        = f'{URL}mnist_sample.tgz'
+    MNIST_TINY          = f'{URL}mnist_tiny.tgz'
+    MNIST_VAR_SIZE_TINY = f'{S3_IMAGE}mnist_var_size_tiny.tgz'
+    PLANET_SAMPLE       = f'{URL}planet_sample.tgz'
+    PLANET_TINY         = f'{URL}planet_tiny.tgz'
+    IMAGENETTE          = f'{S3_IMAGE}imagenette2.tgz'
+    IMAGENETTE_160      = f'{S3_IMAGE}imagenette2-160.tgz'
+    IMAGENETTE_320      = f'{S3_IMAGE}imagenette2-320.tgz'
+    IMAGEWOOF           = f'{S3_IMAGE}imagewoof2.tgz'
+    IMAGEWOOF_160       = f'{S3_IMAGE}imagewoof2-160.tgz'
+    IMAGEWOOF_320       = f'{S3_IMAGE}imagewoof2-320.tgz'
+    IMAGEWANG           = f'{S3_IMAGE}imagewang.tgz'
+    IMAGEWANG_160       = f'{S3_IMAGE}imagewang-160.tgz'
+    IMAGEWANG_320       = f'{S3_IMAGE}imagewang-320.tgz'
+
+    # kaggle competitions download dogs-vs-cats -p {DOGS.absolute()}
+    DOGS = f'{URL}dogscats.tgz'
+
+    # image classification datasets
+    CALTECH_101  = f'{S3_IMAGE}caltech_101.tgz'
+    CARS         = f'{S3_IMAGE}stanford-cars.tgz'
+    CIFAR_100    = f'{S3_IMAGE}cifar100.tgz'
+    CUB_200_2011 = f'{S3_IMAGE}CUB_200_2011.tgz'
+    FLOWERS      = f'{S3_IMAGE}oxford-102-flowers.tgz'
+    FOOD         = f'{S3_IMAGE}food-101.tgz'
+    MNIST        = f'{S3_IMAGE}mnist_png.tgz'
+    PETS         = f'{S3_IMAGE}oxford-iiit-pet.tgz'
+
+    # Image localization datasets
+    BIWI_HEAD_POSE     = f"{S3_IMAGELOC}biwi_head_pose.tgz"
+    CAMVID             = f'{S3_IMAGELOC}camvid.tgz'
+    CAMVID_TINY        = f'{URL}camvid_tiny.tgz'
+    LSUN_BEDROOMS      = f'{S3_IMAGE}bedroom.tgz'
+    PASCAL_2007        = f'{S3_IMAGELOC}pascal_2007.tgz'
+    PASCAL_2012        = f'{S3_IMAGELOC}pascal_2012.tgz'
+
+    # Medical Imaging datasets
+    #SKIN_LESION        = f'{S3_IMAGELOC}skin_lesion.tgz'
+    SIIM_SMALL         = f'{S3_IMAGELOC}siim_small.tgz'
+
+    @staticmethod
+    def path(url='.', c_key='archive'):
+        "Return local path where to download based on `c_key`"
+        fname = url.split('/')[-1]
+        local_path = URLs.LOCAL_PATH / ('models' if c_key == 'models' else 'datasets')/fname
+        if local_path.exists():
+            return local_path
+        return Config()[c_key]/fname
+
+_URL_REGEX = re.compile(
+    r'^(?:http|ftp)s?://' # http:// or https://
+    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
+    r'localhost|' #localhost...
+    r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
+    r'(?::\d+)?' # optional port
+    r'(?:/?|[/?]\S+)$', re.IGNORECASE)
+
+def is_url(url_like):
+    if not isinstance(url_like, str):
+        return False
+    return re.match(_URL_REGEX, url_like) is not None
-Original file line number
+Diff line change
   - decord
   - cython
   - pycocotools
 +  - autocfg
Original file line number	Diff line number	Diff line change
`@@ -170,4 +170,4 @@ def get_dataloader(net, train_dataset, data_shape, batch_size, num_workers):`
`170`	`170`	`# adapting a training blocks in the following examples:`
`171`	`171`	`#`
`172`	`172`	# :download:`Download train_faster_rcnn.py<../../../scripts/detection/faster_rcnn/train_faster_rcnn.py>`
`173`		-# :download:`Download train_yolo3.py<../../../scripts/detection/yolo/train_yolo3.py>`
	`173`	+# :download:`Download train_yolo.py<../../../scripts/detection/yolo/train_yolo.py>`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+"""GluonCV auto"""`
	`2`	`+from .estimators import *`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+"""Data Pipelines"""`
	`2`	`+from .auto_data import url_data, URLs, is_url`