From 0c55a42887fb8e0f61b4224f13cd76348ae993f4 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 23 Mar 2023 17:13:01 +0100
Subject: [PATCH 01/72] DOC: enrich documentation, STY: enforce pep
---
otbtf/dataset.py | 529 ++++++++++++------
.../tensorflow_v2x/fcnn/create_tfrecords.py | 41 +-
.../tensorflow_v2x/fcnn/fcnn_model.py | 164 ++++--
otbtf/examples/tensorflow_v2x/fcnn/helper.py | 19 +-
.../fcnn/train_from_patchesimages.py | 117 ++--
.../fcnn/train_from_tfrecords.py | 33 +-
otbtf/model.py | 219 ++++++--
otbtf/tfrecords.py | 256 ++++++---
otbtf/utils.py | 37 +-
9 files changed, 998 insertions(+), 417 deletions(-)
diff --git a/otbtf/dataset.py b/otbtf/dataset.py
index b7ca2025..4833cb00 100644
--- a/otbtf/dataset.py
+++ b/otbtf/dataset.py
@@ -2,7 +2,7 @@
# ==========================================================================
#
# Copyright 2018-2019 IRSTEA
-# Copyright 2020-2022 INRAE
+# Copyright 2020-2023 INRAE
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,52 +18,65 @@
#
# ==========================================================================*/
"""
-Contains stuff to help working with TensorFlow and geospatial data in the OTBTF framework.
+Contains stuff to help working with TensorFlow and geospatial data in the
+OTBTF framework.
"""
-import threading
+import logging
import multiprocessing
+import threading
import time
-import logging
from abc import ABC, abstractmethod
+
import numpy as np
import tensorflow as tf
-from otbtf.utils import read_as_np_arr, gdal_open
+from typing import Any, List, Dict, Type, Callable
+
from otbtf.tfrecords import TFRecords
+from otbtf.utils import read_as_np_arr, gdal_open
-# -------------------------------------------------- Buffer class ------------------------------------------------------
+# --------------------------- Buffer class ------------------------------------
class Buffer:
"""
Used to store and access list of objects
"""
- def __init__(self, max_length):
+ def __init__(self, max_length: int):
+ """
+ Params:
+ max_length: length of the buffer
+ """
self.max_length = max_length
self.container = []
- def size(self):
+ def size(self) -> int:
"""
- Returns the buffer size
+ Returns:
+ the buffer size
"""
return len(self.container)
- def add(self, new_element):
+ def add(self, new_element: Any):
"""
- Add an element in the buffer
- :param new_element: new element to add
+ Add an element in the buffer.
+
+ Params:
+ new_element: new element to add
+
"""
self.container.append(new_element)
assert self.size() <= self.max_length
- def is_complete(self):
+ def is_complete(self) -> bool:
"""
- Return True if the buffer is at full capacity
+ Returns:
+ True if the buffer is full, False else
"""
return self.size() == self.max_length
-# ---------------------------------------------- PatchesReaderBase class -----------------------------------------------
+# ------------------------ PatchesReaderBase class ----------------------------
class PatchesReaderBase(ABC):
"""
@@ -71,39 +84,51 @@ class PatchesReaderBase(ABC):
"""
@abstractmethod
- def get_sample(self, index):
+ def get_sample(self, index: int) -> Any:
"""
Return one sample.
- :return One sample instance, whatever the sample structure is (dict, numpy array, ...)
+
+ Params:
+ index: sample index
+
+ Returns:
+ One sample instance, whatever the sample structure is (dict, numpy
+ array, ...)
"""
@abstractmethod
def get_stats(self) -> dict:
"""
Compute some statistics for each source.
- Depending if streaming is used, the statistics are computed directly in memory, or chunk-by-chunk.
-
- :return a dict having the following structure:
- {
- "src_key_0":
- {"min": np.array([...]),
- "max": np.array([...]),
- "mean": np.array([...]),
- "std": np.array([...])},
- ...,
- "src_key_M":
- {"min": np.array([...]),
- "max": np.array([...]),
- "mean": np.array([...]),
- "std": np.array([...])},
- }
+ Depending on if streaming is used, the statistics are computed
+ directly in memory, or chunk-by-chunk.
+
+ Returns:
+ a dict having the following structure:
+ {
+ "src_key_0":
+ {"min": np.array([...]),
+ "max": np.array([...]),
+ "mean": np.array([...]),
+ "std": np.array([...])},
+ ...,
+ "src_key_M":
+ {"min": np.array([...]),
+ "max": np.array([...]),
+ "mean": np.array([...]),
+ "std": np.array([...])},
+ }
+
"""
@abstractmethod
- def get_size(self):
+ def get_size(self) -> int:
"""
Returns the total number of samples
- :return: number of samples (int)
+
+ Returns:
+ number of samples (int)
+
"""
@@ -113,52 +138,90 @@ class PatchesImagesReader(PatchesReaderBase):
"""
This class provides a read access to a set of patches images.
- A patches image is an image of patches stacked in rows, as produced from the OTBTF "PatchesExtraction"
- application, and is stored in a raster format (e.g. GeoTiff).
- A source can be a particular domain in which the patches are extracted (remember that in OTBTF applications,
- the number of sources is controlled by the OTB_TF_NSOURCES environment variable).
+ A patches image is an image of patches stacked in rows, as produced from
+ the OTBTF "PatchesExtraction" application, and is stored in a raster
+ format (e.g. GeoTiff).
+ A source can be a particular domain in which the patches are extracted
+ (remember that in OTBTF applications, the number of sources is controlled
+ by the `OTB_TF_NSOURCES` environment variable).
This class enables to use:
- multiple sources
- multiple patches images per source
- Each patch can be independently accessed using the get_sample(index) function, with index in [0, self.size),
- self.size being the total number of patches (must be the same for each sources).
+ Each patch can be independently accessed using the get_sample(index)
+ function, with index in [0, self.size), self.size being the total number
+ of patches (must be the same for each sources).
+
+ See `PatchesReaderBase`.
- :see PatchesReaderBase
"""
- def __init__(self, filenames_dict, use_streaming=False, scalar_dict=None):
- """
- :param filenames_dict: A dict() structured as follow:
- {src_name1: [src1_patches_image_1.tif, ..., src1_patches_image_N.tif],
- src_name2: [src2_patches_image_1.tif, ..., src2_patches_image_N.tif],
- ...
- src_nameM: [srcM_patches_image_1.tif, ..., srcM_patches_image_N.tif]}
- :param use_streaming: if True, the patches are read on the fly from the disc, nothing is kept in memory.
- :param scalar_dict: (optional) a dict containing list of scalars (int, float, str) as follow:
- {scalar_name1: ["value_1", ..., "value_N"],
- scalar_name2: [value_1, ..., value_N],
- ...
- scalar_nameM: [value1, ..., valueN]}
+ def __init__(
+ self,
+ filenames_dict: Dict[str, List[str]],
+ use_streaming: bool = False,
+ scalar_dict: Dict[str, List[Any]] = None
+ ):
+ """
+ Params:
+ filenames_dict: A dict() structured as follow:
+ {
+ src_name1: [
+ src1_patches_image_1.tif, ..., src1_patches_image_N.tif
+ ],
+ src_name2: [
+ src2_patches_image_1.tif, ..., src2_patches_image_N.tif
+ ],
+ ...
+ src_nameM: [
+ srcM_patches_image_1.tif, ..., srcM_patches_image_N.tif
+ ]
+ }
+ use_streaming: if True, the patches are read on the fly from the
+ disc, nothing is kept in memory. Else, everything is packed in
+ memory.
+ scalar_dict: (optional) a dict containing list of scalars (int,
+ float, str) as follow:
+ {
+ scalar_name1: ["value_1", ..., "value_N"],
+ scalar_name2: [value_1, ..., value_N],
+ ...
+ scalar_nameM: [value1, ..., valueN]
+ }
+
"""
assert len(filenames_dict.values()) > 0
# gdal_ds dict
- self.gdal_ds = {key: [gdal_open(src_fn) for src_fn in src_fns] for key, src_fns in filenames_dict.items()}
+ self.gdal_ds = {
+ key: [gdal_open(src_fn) for src_fn in src_fns]
+ for key, src_fns in filenames_dict.items()
+ }
# streaming on/off
self.use_streaming = use_streaming
# Scalar dict (e.g. for metadata)
# If the scalars are not numpy.ndarray, convert them
- self.scalar_dict = {key: [i if isinstance(i, np.ndarray) else np.asarray(i) for i in scalars]
- for key, scalars in scalar_dict.items()} if scalar_dict else {}
+ self.scalar_dict = {
+ key: [
+ i if isinstance(i, np.ndarray) else np.asarray(i)
+ for i in scalars
+ ]
+ for key, scalars in scalar_dict.items()
+ } if scalar_dict else {}
# check number of patches in each sources
- if len({len(ds_list) for ds_list in list(self.gdal_ds.values()) + list(self.scalar_dict.values())}) != 1:
- raise Exception("Each source must have the same number of patches images")
+ if len({
+ len(ds_list)
+ for ds_list in
+ list(self.gdal_ds.values()) + list(self.scalar_dict.values())
+ }) != 1:
+ raise Exception(
+ "Each source must have the same number of patches images"
+ )
# gdal_ds check
nb_of_patches = {key: 0 for key in self.gdal_ds}
@@ -170,20 +233,33 @@ class PatchesImagesReader(PatchesReaderBase):
self.nb_of_channels[src_key] = gdal_ds.RasterCount
else:
if self.nb_of_channels[src_key] != gdal_ds.RasterCount:
- raise Exception("All patches images from one source must have the same number of channels!"
- f"Error happened for source: {src_key}")
+ raise Exception(
+ "All patches images from one source must have the "
+ "same number of channels! "
+ f"Error happened for source: {src_key}"
+ )
if len(set(nb_of_patches.values())) != 1:
- raise Exception(f"Sources must have the same number of patches! Number of patches: {nb_of_patches}")
+ raise Exception(
+ "Sources must have the same number of patches! "
+ f"Number of patches: {nb_of_patches}"
+ )
# gdal_ds sizes
src_key_0 = list(self.gdal_ds)[0] # first key
- self.ds_sizes = [self._get_nb_of_patches(ds) for ds in self.gdal_ds[src_key_0]]
+ self.ds_sizes = [
+ self._get_nb_of_patches(ds)
+ for ds in self.gdal_ds[src_key_0]
+ ]
self.size = sum(self.ds_sizes)
# if use_streaming is False, we store in memory all patches images
if not self.use_streaming:
- self.patches_buffer = {src_key: np.concatenate([read_as_np_arr(ds) for ds in src_ds], axis=0) for
- src_key, src_ds in self.gdal_ds.items()}
+ self.patches_buffer = {
+ src_key: np.concatenate([
+ read_as_np_arr(ds) for ds in src_ds
+ ], axis=0)
+ for src_key, src_ds in self.gdal_ds.items()
+ }
def _get_ds_and_offset_from_index(self, index):
offset = index
@@ -211,48 +287,73 @@ class PatchesImagesReader(PatchesReaderBase):
return np.transpose(buffer, axes=(1, 2, 0))
return np.expand_dims(buffer, axis=2)
- def get_sample(self, index):
+ def get_sample(self, index: int) -> Dict[str, np.array]:
"""
Return one sample of the dataset.
- :param index: the sample index. Must be in the [0, self.size) range.
- :return: The sample is stored in a dict() with the following structure:
- {"src_key_0": np.array((psz_y_0, psz_x_0, nb_ch_0)),
- "src_key_1": np.array((psz_y_1, psz_x_1, nb_ch_1)),
- ...
- "src_key_M": np.array((psz_y_M, psz_x_M, nb_ch_M))}
+
+ Params:
+ index: the sample index. Must be in the [0, self.size) range.
+
+ Returns:
+ The sample is stored in a dict() with the following structure:
+ {
+ "src_key_0": np.array((psz_y_0, psz_x_0, nb_ch_0)),
+ "src_key_1": np.array((psz_y_1, psz_x_1, nb_ch_1)),
+ ...
+ "src_key_M": np.array((psz_y_M, psz_x_M, nb_ch_M))
+ }
+
"""
assert index >= 0
assert index < self.size
i, offset = self._get_ds_and_offset_from_index(index)
- res = {src_key: scalar[i] for src_key, scalar in self.scalar_dict.items()}
+ res = {
+ src_key: scalar[i]
+ for src_key, scalar in self.scalar_dict.items()
+ }
if not self.use_streaming:
- res.update({src_key: arr[index, :, :, :] for src_key, arr in self.patches_buffer.items()})
+ res.update({
+ src_key: arr[index, :, :, :]
+ for src_key, arr in self.patches_buffer.items()
+ })
else:
- res.update({src_key: self._read_extract_as_np_arr(self.gdal_ds[src_key][i], offset)
- for src_key in self.gdal_ds})
+ res.update({
+ src_key: self._read_extract_as_np_arr(
+ self.gdal_ds[src_key][i], offset
+ )
+ for src_key in self.gdal_ds
+ })
return res
- def get_stats(self):
+ def get_stats(self) -> Dict[str, List[float]]:
"""
Compute some statistics for each source.
- When streaming is used, chunk-by-chunk. Else, the statistics are computed directly in memory.
+ When streaming is used, chunk-by-chunk. Else, the statistics are
+ computed directly in memory.
- :return statistics dict
+ Returns:
+ statistics dict
"""
logging.info("Computing stats")
if not self.use_streaming:
axis = (0, 1, 2) # (row, col)
- stats = {src_key: {"min": np.amin(patches_buffer, axis=axis),
- "max": np.amax(patches_buffer, axis=axis),
- "mean": np.mean(patches_buffer, axis=axis),
- "std": np.std(patches_buffer, axis=axis)} for src_key, patches_buffer in
- self.patches_buffer.items()}
+ stats = {
+ src_key: {
+ "min": np.amin(patches_buffer, axis=axis),
+ "max": np.amax(patches_buffer, axis=axis),
+ "mean": np.mean(patches_buffer, axis=axis),
+ "std": np.std(patches_buffer, axis=axis)
+ }
+ for src_key, patches_buffer in self.patches_buffer.items()
+ }
else:
axis = (0, 1) # (row, col)
def _filled(value):
- return {src_key: value * np.ones((self.nb_of_channels[src_key])) for src_key in self.gdal_ds}
+ return {
+ src_key: value * np.ones((self.nb_of_channels[src_key]))
+ for src_key in self.gdal_ds}
_maxs = _filled(0.0)
_mins = _filled(float("inf"))
@@ -262,21 +363,41 @@ class PatchesImagesReader(PatchesReaderBase):
sample = self.get_sample(index=index)
for src_key, np_arr in sample.items():
rnumel = 1.0 / float(np_arr.shape[0] * np_arr.shape[1])
- _mins[src_key] = np.minimum(np.amin(np_arr, axis=axis).flatten(), _mins[src_key])
- _maxs[src_key] = np.maximum(np.amax(np_arr, axis=axis).flatten(), _maxs[src_key])
- _sums[src_key] += rnumel * np.sum(np_arr, axis=axis).flatten()
- _sqsums[src_key] += rnumel * np.sum(np.square(np_arr), axis=axis).flatten()
+ _mins[src_key] = np.minimum(
+ np.amin(np_arr, axis=axis).flatten(), _mins[src_key]
+ )
+ _maxs[src_key] = np.maximum(
+ np.amax(np_arr, axis=axis).flatten(), _maxs[src_key]
+ )
+ _sums[src_key] += rnumel * np.sum(
+ np_arr, axis=axis
+ ).flatten()
+ _sqsums[src_key] += rnumel * np.sum(
+ np.square(np_arr), axis=axis
+ ).flatten()
rsize = 1.0 / float(self.size)
- stats = {src_key: {"min": _mins[src_key],
- "max": _maxs[src_key],
- "mean": rsize * _sums[src_key],
- "std": np.sqrt(rsize * _sqsums[src_key] - np.square(rsize * _sums[src_key]))
- } for src_key in self.gdal_ds}
+ stats = {
+ src_key: {
+ "min": _mins[src_key],
+ "max": _maxs[src_key],
+ "mean": rsize * _sums[src_key],
+ "std": np.sqrt(
+ rsize * _sqsums[src_key] - np.square(
+ rsize * _sums[src_key]
+ )
+ )
+ }
+ for src_key in self.gdal_ds
+ }
logging.info("Stats: %s", stats)
return stats
- def get_size(self):
+ def get_size(self) -> int:
+ """
+ Returns:
+ size
+ """
return self.size
@@ -292,14 +413,18 @@ class IteratorBase(ABC):
pass
-# ---------------------------------------------- RandomIterator class --------------------------------------------------
+# ---------------------------- RandomIterator class ---------------------------
class RandomIterator(IteratorBase):
"""
Pick a random number in the [0, handler.size) range.
"""
- def __init__(self, patches_reader):
+ def __init__(self, patches_reader: PatchesReaderBase):
+ """
+ Params:
+ patches_reader: patches reader
+ """
super().__init__(patches_reader=patches_reader)
self.indices = np.arange(0, patches_reader.get_size())
self._shuffle()
@@ -326,18 +451,24 @@ class RandomIterator(IteratorBase):
class Dataset:
"""
Handles the "mining" of patches.
- This class has a thread that extract tuples from the readers, while ensuring the access of already gathered tuples.
+ This class has a thread that extract tuples from the readers, while
+ ensuring the access of already gathered tuples.
+
+ See `PatchesReaderBase` and `Buffer`
- :see PatchesReaderBase
- :see Buffer
"""
- def __init__(self, patches_reader: PatchesReaderBase = None, buffer_length: int = 128,
- Iterator=RandomIterator, max_nb_of_samples=None):
+ def __init__(
+ self,
+ patches_reader: PatchesReaderBase = None,
+ buffer_length: int = 128,
+ iterator_cls: Type[IteratorBase] = RandomIterator,
+ max_nb_of_samples: int = None
+ ):
"""
:param patches_reader: The patches reader instance
:param buffer_length: The number of samples that are stored in the buffer
- :param Iterator: The iterator class used to generate the sequence of patches indices.
+ :param iterator_cls: The iterator class used to generate the sequence of patches indices.
:param max_nb_of_samples: Optional, max number of samples to consider
"""
# patches reader
@@ -345,14 +476,15 @@ class Dataset:
# If necessary, limit the nb of samples
logging.info('Number of samples: %s', self.patches_reader.get_size())
- if max_nb_of_samples and self.patches_reader.get_size() > max_nb_of_samples:
+ if max_nb_of_samples and \
+ self.patches_reader.get_size() > max_nb_of_samples:
logging.info('Reducing number of samples to %s', max_nb_of_samples)
self.size = max_nb_of_samples
else:
self.size = self.patches_reader.get_size()
# iterator
- self.iterator = Iterator(patches_reader=self.patches_reader)
+ self.iterator = iterator_cls(patches_reader=self.patches_reader)
# Get patches sizes and type, of the first sample of the first tile
self.output_types = dict()
@@ -378,39 +510,60 @@ class Dataset:
self._dump()
# Prepare tf dataset for one epoch
- self.tf_dataset = tf.data.Dataset.from_generator(self._generator,
- output_types=self.output_types,
- output_shapes=self.output_shapes).repeat(1)
-
- def to_tfrecords(self, output_dir, n_samples_per_shard=100, drop_remainder=True):
+ self.tf_dataset = tf.data.Dataset.from_generator(
+ self._generator,
+ output_types=self.output_types,
+ output_shapes=self.output_shapes
+ ).repeat(1)
+
+ def to_tfrecords(
+ self,
+ output_dir: str,
+ n_samples_per_shard: int = 100,
+ drop_remainder: bool = True
+ ):
"""
Save the dataset into TFRecord files
- :param output_dir: output directory
- :param n_samples_per_shard: number of samples per TFRecord file
- :param drop_remainder: drop remainder samples
+ Params:
+ output_dir: output directory
+ n_samples_per_shard: number of samples per TFRecord file
+ drop_remainder: drop remaining samples
+
"""
tfrecord = TFRecords(output_dir)
- tfrecord.ds2tfrecord(self, n_samples_per_shard=n_samples_per_shard, drop_remainder=drop_remainder)
+ tfrecord.ds2tfrecord(
+ self,
+ n_samples_per_shard=n_samples_per_shard,
+ drop_remainder=drop_remainder
+ )
- def get_stats(self) -> dict:
+ def get_stats(self) -> Dict[str, List[float]]:
"""
Compute dataset statistics
- :return: the dataset statistics, computed by the patches reader
+ Return:
+ the dataset statistics, computed by the patches reader
+
"""
with self.mining_lock:
return self.patches_reader.get_stats()
- def read_one_sample(self):
+ def read_one_sample(self) -> Dict[str, Any]:
"""
Read one element of the consumer_buffer
- The lock is used to prevent different threads to read and update the internal counter concurrently
+ The lock is used to prevent different threads to read and update the
+ internal counter concurrently
+
+ Return:
+ one sample
+
"""
with self.read_lock:
output = None
if self.consumer_buffer_pos < self.consumer_buffer.max_length:
- output = self.consumer_buffer.container[self.consumer_buffer_pos]
+ output = self.consumer_buffer.container[
+ self.consumer_buffer_pos]
self.consumer_buffer_pos += 1
if self.consumer_buffer_pos == self.consumer_buffer.max_length:
self._dump()
@@ -419,7 +572,9 @@ class Dataset:
def _dump(self):
"""
- This function dumps the miner_buffer into the consumer_buffer, and restart the miner_thread
+ This function dumps the miner_buffer into the consumer_buffer, and
+ restarts the miner_thread
+
"""
# Wait for miner to finish his job
date_t = time.time()
@@ -439,6 +594,7 @@ class Dataset:
"""
This function collects samples.
It is threaded by the miner_thread.
+
"""
# Fill the miner_container until it's full
while not self.miner_buffer.is_complete():
@@ -447,7 +603,7 @@ class Dataset:
new_sample = self.patches_reader.get_sample(index=index)
self.miner_buffer.add(new_sample)
- def _summon_miner_thread(self):
+ def _summon_miner_thread(self) -> threading.Thread:
"""
Create and starts the thread for the data collect
"""
@@ -462,51 +618,81 @@ class Dataset:
for _ in range(self.size):
yield self.read_one_sample()
- def get_tf_dataset(self, batch_size, drop_remainder=True, preprocessing_fn=None, targets_keys=None):
+ def get_tf_dataset(
+ self,
+ batch_size: int,
+ drop_remainder: bool = True,
+ preprocessing_fn: Callable = None,
+ targets_keys: List[str] = None
+ ) -> tf.Dataset:
"""
Returns a TF dataset, ready to be used with the provided batch size
- :param batch_size: the batch size
- :param drop_remainder: drop incomplete batches
- :param preprocessing_fn: Optional. A preprocessing function that takes input examples as args and returns the
- preprocessed input examples. Typically, examples are composed of model inputs and
- targets. Model inputs and model targets must be computed accordingly to (1) what the
- model outputs and (2) what training loss needs. For instance, for a classification
- problem, the model will likely output the softmax, or activation neurons, for each
- class, and the cross entropy loss requires labels in one hot encoding. In this case,
- the preprocessing_fn has to transform the labels values (integer ranging from
- [0, n_classes]) in one hot encoding (vector of 0 and 1 of length n_classes). The
- preprocessing_fn should not implement such things as radiometric transformations from
- input to input_preprocessed, because those are performed inside the model itself
- (see `otbtf.ModelBase.normalize_inputs()`).
- :param targets_keys: Optional. When provided, the dataset returns a tuple of dicts (inputs_dict, target_dict) so
- it can be straightforwardly used with keras models objects.
- :return: The TF dataset
+
+ Params:
+ batch_size: the batch size
+ drop_remainder: drop incomplete batches
+ preprocessing_fn: An optional preprocessing function that takes
+ input examples as args and returns the preprocessed input
+ examples. Typically, examples are composed of model inputs and
+ targets. Model inputs and model targets must be computed
+ accordingly to (1) what the model outputs and (2) what
+ training loss needs. For instance, for a classification
+ problem, the model will likely output the softmax, or
+ activation neurons, for each class, and the cross entropy loss
+ requires labels in one hot encoding. In this case, the
+ preprocessing_fn has to transform the labels values (integer
+ ranging from [0, n_classes]) in one hot encoding (vector of 0
+ and 1 of length n_classes). The preprocessing_fn should not
+ implement such things as radiometric transformations from
+ input to input_preprocessed, because those are performed
+ inside the model itself (see
+ `otbtf.ModelBase.normalize_inputs()`).
+ targets_keys: Optional. When provided, the dataset returns a tuple
+ of dicts (inputs_dict, target_dict) so it can be
+ straightforwardly used with keras models objects.
+
+ Returns:
+ The TF dataset
+
"""
if 2 * batch_size >= self.miner_buffer.max_length:
- logging.warning("Batch size is %s but dataset buffer has %s elements. Consider using a larger dataset "
- "buffer to avoid I/O bottleneck", batch_size, self.miner_buffer.max_length)
- tf_ds = self.tf_dataset.map(preprocessing_fn) if preprocessing_fn else self.tf_dataset
+ logging.warning(
+ "Batch size is %s but dataset buffer has %s elements. "
+ "Consider using a larger dataset buffer to avoid I/O "
+ "bottleneck", batch_size, self.miner_buffer.max_length
+ )
+ tf_ds = self.tf_dataset.map(preprocessing_fn) \
+ if preprocessing_fn else self.tf_dataset
if targets_keys:
def _split_input_and_target(example):
# Differentiating inputs and outputs for keras
- inputs = {key: value for (key, value) in example.items() if key not in targets_keys}
- targets = {key: value for (key, value) in example.items() if key in targets_keys}
+ inputs = {
+ key: value for (key, value) in example.items()
+ if key not in targets_keys
+ }
+ targets = {
+ key: value for (key, value) in example.items()
+ if key in targets_keys
+ }
return inputs, targets
tf_ds = tf_ds.map(_split_input_and_target)
return tf_ds.batch(batch_size, drop_remainder=drop_remainder)
- def get_total_wait_in_seconds(self):
+ def get_total_wait_in_seconds(self) -> int:
"""
- Returns the number of seconds during which the data gathering was delayed because of I/O bottleneck
- :return: duration in seconds
+ Returns the number of seconds during which the data gathering was
+ delayed because I/O bottleneck
+
+ Returns:
+ duration in seconds
"""
return self.tot_wait
-# ----------------------------------------- DatasetFromPatchesImages class ---------------------------------------------
+# --------------------- DatasetFromPatchesImages class ------------------------
class DatasetFromPatchesImages(Dataset):
"""
@@ -516,19 +702,38 @@ class DatasetFromPatchesImages(Dataset):
:see Dataset
"""
- def __init__(self, filenames_dict, use_streaming=False, buffer_length: int = 128,
- Iterator=RandomIterator):
- """
- :param filenames_dict: A dict() structured as follow:
- {src_name1: [src1_patches_image1, ..., src1_patches_imageN1],
- src_name2: [src2_patches_image2, ..., src2_patches_imageN2],
- ...
- src_nameM: [srcM_patches_image1, ..., srcM_patches_imageNM]}
- :param use_streaming: if True, the patches are read on the fly from the disc, nothing is kept in memory.
- :param buffer_length: The number of samples that are stored in the buffer (used when "use_streaming" is True).
- :param Iterator: The iterator class used to generate the sequence of patches indices.
+ def __init__(
+ self,
+ filenames_dict: Dict[str, List[str]],
+ use_streaming: bool = False,
+ buffer_length: int = 128,
+ iterator_cls=RandomIterator
+ ):
+ """
+ Params:
+ filenames_dict: A dict() structured as follow:
+ {
+ src_name1: [src1_patches_image1, ..., src1_patches_imageN1],
+ src_name2: [src2_patches_image2, ..., src2_patches_imageN2],
+ ...
+ src_nameM: [srcM_patches_image1, ..., srcM_patches_imageNM]
+ }
+ use_streaming: if True, the patches are read on the fly from the disc,
+ nothing is kept in memory.
+ buffer_length: The number of samples that are stored in the buffer
+ (used when "use_streaming" is True).
+ iterator_cls: The iterator class used to generate the sequence of
+ patches indices.
+
"""
# patches reader
- patches_reader = PatchesImagesReader(filenames_dict=filenames_dict, use_streaming=use_streaming)
-
- super().__init__(patches_reader=patches_reader, buffer_length=buffer_length, Iterator=Iterator)
+ patches_reader = PatchesImagesReader(
+ filenames_dict=filenames_dict,
+ use_streaming=use_streaming
+ )
+
+ super().__init__(
+ patches_reader=patches_reader,
+ buffer_length=buffer_length,
+ iterator_cls=iterator_cls
+ )
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
index 51043ef1..605c28aa 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
@@ -1,17 +1,35 @@
"""
-This example shows how to convert patches-images (like the ones generated from the `PatchesExtraction`)
-into TFRecords files.
+This example shows how to convert patches-images (like the ones generated from
+the `PatchesExtraction`) into TFRecords files.
"""
import argparse
from pathlib import Path
-from otbtf.examples.tensorflow_v2x.fcnn import helper
+
from otbtf import DatasetFromPatchesImages
+from otbtf.examples.tensorflow_v2x.fcnn import helper
-parser = argparse.ArgumentParser(description="Converts patches-images into TFRecords")
-parser.add_argument("--xs", required=True, nargs="+", default=[], help="A list of patches-images for the XS image")
-parser.add_argument("--labels", required=True, nargs="+", default=[],
- help="A list of patches-images for the labels")
-parser.add_argument("--outdir", required=True, help="Output dir for TFRecords files")
+parser = argparse.ArgumentParser(
+ description="Converts patches-images into TFRecords"
+)
+parser.add_argument(
+ "--xs",
+ required=True,
+ nargs="+",
+ default=[],
+ help="A list of patches-images for the XS image"
+)
+parser.add_argument(
+ "--labels",
+ required=True,
+ nargs="+",
+ default=[],
+ help="A list of patches-images for the labels"
+)
+parser.add_argument(
+ "--outdir",
+ required=True,
+ help="Output dir for TFRecords files"
+)
def create_tfrecords(params):
@@ -28,7 +46,12 @@ def create_tfrecords(params):
outdir.mkdir(exist_ok=True)
# Create dataset from the filename dict
- dataset = DatasetFromPatchesImages(filenames_dict={"input_xs_patches": patches, "labels_patches": labels})
+ dataset = DatasetFromPatchesImages(
+ filenames_dict={
+ "input_xs_patches": patches,
+ "labels_patches": labels
+ }
+ )
# Convert the dataset into TFRecords
dataset.to_tfrecords(output_dir=params.outdir, drop_remainder=False)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 95f2d017..804744c2 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -1,15 +1,30 @@
"""
Implementation of a small U-Net like model
"""
-from otbtf.model import ModelBase
-import tensorflow as tf
import logging
-logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
+import tensorflow as tf
+
+from otbtf.model import ModelBase
+
+logging.basicConfig(
+ format='%(asctime)s %(levelname)-8s %(message)s',
+ level=logging.INFO,
+ datefmt='%Y-%m-%d %H:%M:%S'
+)
+
+# Number of classes estimated by the model
N_CLASSES = 2
-INPUT_NAME = "input_xs" # name of the input in the `FCNNModel` instance, also name of the input node in the SavedModel
-TARGET_NAME = "predictions" # name of the output in the `FCNNModel` instance
-OUTPUT_SOFTMAX_NAME = "predictions_softmax_tensor" # name (prefix) of the output node in the SavedModel
+
+# Name of the input in the `FCNNModel` instance, also name of the input node
+# in the SavedModel
+INPUT_NAME = "input_xs"
+
+# Name of the output in the `FCNNModel` instance
+TARGET_NAME = "predictions"
+
+# Name (prefix) of the output node in the SavedModel
+OUTPUT_SOFTMAX_NAME = "predictions_softmax_tensor"
class FCNNModel(ModelBase):
@@ -17,42 +32,65 @@ class FCNNModel(ModelBase):
A Simple Fully Convolutional U-Net like model
"""
- def normalize_inputs(self, inputs):
+ def normalize_inputs(self, inputs: dict):
"""
Inherits from `ModelBase`
- The model will use this function internally to normalize its inputs, before applying the `get_outputs()`
- function that actually builds the operations graph (convolutions, etc).
- This function will hence work at training time and inference time.
+ The model will use this function internally to normalize its inputs,
+ before applying `get_outputs()` that actually builds the operations
+ graph (convolutions, etc). This function will hence work at training
+ time and inference time.
- In this example, we assume that we have an input 12 bits multispectral image with values ranging from
- [0, 10000], that we process using a simple stretch to roughly match the [0, 1] range.
+ In this example, we assume that we have an input 12 bits multispectral
+ image with values ranging from [0, 10000], that we process using a
+ simple stretch to roughly match the [0, 1] range.
- :param inputs: dict of inputs
- :return: dict of normalized inputs, ready to be used from the `get_outputs()` function of the model
+ Params:
+ inputs: dict of inputs
+
+ Returns:
+ dict of normalized inputs, ready to be used from `get_outputs()`
"""
return {INPUT_NAME: tf.cast(inputs[INPUT_NAME], tf.float32) * 0.0001}
- def get_outputs(self, normalized_inputs):
+ def get_outputs(self, normalized_inputs: dict) -> dict:
"""
Inherits from `ModelBase`
- This small model produces an output which has the same physical spacing as the input.
- The model generates [1 x 1 x N_CLASSES] output pixel for [32 x 32 x <nb channels>] input pixels.
+ This small model produces an output which has the same physical
+ spacing as the input. The model generates [1 x 1 x N_CLASSES] output
+ pixel for [32 x 32 x <nb channels>] input pixels.
+
+ Params:
+ normalized_inputs: dict of normalized inputs
- :param normalized_inputs: dict of normalized inputs`
- :return: activation values
+ Returns:
+ dict of model outputs
"""
norm_inp = normalized_inputs[INPUT_NAME]
def _conv(inp, depth, name):
- return tf.keras.layers.Conv2D(filters=depth, kernel_size=3, strides=2, activation="relu",
- padding="same", name=name)(inp)
+ conv_op = tf.keras.layers.Conv2D(
+ filters=depth,
+ kernel_size=3,
+ strides=2,
+ activation="relu",
+ padding="same",
+ name=name
+ )
+ return conv_op(inp)
def _tconv(inp, depth, name, activation="relu"):
- return tf.keras.layers.Conv2DTranspose(filters=depth, kernel_size=3, strides=2, activation=activation,
- padding="same", name=name)(inp)
+ tconv_op = tf.keras.layers.Conv2DTranspose(
+ filters=depth,
+ kernel_size=3,
+ strides=2,
+ activation=activation,
+ padding="same",
+ name=name
+ )
+ return tconv_op(inp)
out_conv1 = _conv(norm_inp, 16, "conv1")
out_conv2 = _conv(out_conv1, 32, "conv2")
@@ -63,63 +101,85 @@ class FCNNModel(ModelBase):
out_tconv3 = _tconv(out_tconv2, 16, "tconv3") + out_conv1
out_tconv4 = _tconv(out_tconv3, N_CLASSES, "classifier", None)
- # Generally it is a good thing to name the final layers of the network (i.e. the layers of which outputs are
- # returned from the `MyModel.get_output()` method).
- # Indeed this enables to retrieve them for inference time, using their name.
- # In case your forgot to name the last layers, it is still possible to look at the model outputs using the
- # `saved_model_cli show --dir /path/to/your/savedmodel --all` command.
+ # Generally it is a good thing to name the final layers of the network
+ # (i.e. the layers of which outputs are returned from
+ # `MyModel.get_output()`). Indeed this enables to retrieve them for
+ # inference time, using their name. In case your forgot to name the
+ # last layers, it is still possible to look at the model outputs using
+ # the `saved_model_cli show --dir /path/to/your/savedmodel --all`
+ # command.
#
- # Do not confuse **the name of the output layers** (i.e. the "name" property of the tf.keras.layer that is used
- # to generate an output tensor) and **the key of the output tensor**, in the dict returned from the
- # `MyModel.get_output()` method. They are two identifiers with a different purpose:
- # - the output layer name is used only at inference time, to identify the output tensor from which generate
- # the output image,
- # - the output tensor key identifies the output tensors, mainly to fit the targets to model outputs during
- # training process, but it can also be used to access the tensors as tf/keras objects, for instance to
- # display previews images in TensorBoard.
- predictions = tf.keras.layers.Softmax(name=OUTPUT_SOFTMAX_NAME)(out_tconv4)
+ # Do not confuse **the name of the output layers** (i.e. the "name"
+ # property of the tf.keras.layer that is used to generate an output
+ # tensor) and **the key of the output tensor**, in the dict returned
+ # from `MyModel.get_output()`. They are two identifiers with a
+ # different purpose:
+ # - the output layer name is used only at inference time, to identify
+ # the output tensor from which generate the output image,
+ # - the output tensor key identifies the output tensors, mainly to
+ # fit the targets to model outputs during training process, but it
+ # can also be used to access the tensors as tf/keras objects, for
+ # instance to display previews images in TensorBoard.
+ softmax_op = tf.keras.layers.Softmax(name=OUTPUT_SOFTMAX_NAME)
+ predictions = softmax_op(out_tconv4)
return {TARGET_NAME: predictions}
-def dataset_preprocessing_fn(examples):
+def dataset_preprocessing_fn(examples: dict):
"""
Preprocessing function for the training dataset.
This function is only used at training time, to put the data in the expected format for the training step.
DO NOT USE THIS FUNCTION TO NORMALIZE THE INPUTS ! (see `otbtf.ModelBase.normalize_inputs` for that).
Note that this function is not called here, but in the code that prepares the datasets.
- :param examples: dict for examples (i.e. inputs and targets stored in a single dict)
- :return: preprocessed examples
+ Params:
+ examples: dict for examples (i.e. inputs and targets stored in a single
+ dict)
+
+ Returns:
+ preprocessed examples
+
"""
def _to_categorical(x):
- return tf.one_hot(tf.squeeze(tf.cast(x, tf.int32), axis=-1), depth=N_CLASSES)
+ return tf.one_hot(
+ tf.squeeze(tf.cast(x, tf.int32), axis=-1), depth=N_CLASSES
+ )
- return {INPUT_NAME: examples["input_xs_patches"],
- TARGET_NAME: _to_categorical(examples["labels_patches"])}
+ return {
+ INPUT_NAME: examples["input_xs_patches"],
+ TARGET_NAME: _to_categorical(examples["labels_patches"])
+ }
def train(params, ds_train, ds_valid, ds_test):
"""
Create, train, and save the model.
- :param params: contains batch_size, learning_rate, nb_epochs, and model_dir
- :param ds_train: training dataset
- :param ds_valid: validation dataset
- :param ds_test: testing dataset
+ Params:
+ params: contains batch_size, learning_rate, nb_epochs, and model_dir
+ ds_train: training dataset
+ ds_valid: validation dataset
+ ds_test: testing dataset
+
"""
strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
with strategy.scope():
- # Model instantiation. Note that the normalize_fn is now part of the model
- # It is mandatory to instantiate the model inside the strategy scope.
+ # Model instantiation. Note that the normalize_fn is now part of the
+ # model. It is mandatory to instantiate the model inside the strategy
+ # scope.
model = FCNNModel(dataset_element_spec=ds_train.element_spec)
# Compile the model
- model.compile(loss=tf.keras.losses.CategoricalCrossentropy(),
- optimizer=tf.keras.optimizers.Adam(learning_rate=params.learning_rate),
- metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
+ model.compile(
+ loss=tf.keras.losses.CategoricalCrossentropy(),
+ optimizer=tf.keras.optimizers.Adam(
+ learning_rate=params.learning_rate
+ ),
+ metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()]
+ )
# Summarize the model (in CLI)
model.summary()
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/helper.py b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
index aea3a0ac..bcc9036a 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/helper.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
@@ -11,17 +11,26 @@ def base_parser():
:return: argparse.ArgumentParser instance
"""
parser = argparse.ArgumentParser(description="Train a FCNN model")
- parser.add_argument("--batch_size", type=int, default=8, help="Batch size")
- parser.add_argument("--learning_rate", type=float, default=0.0001, help="Learning rate")
- parser.add_argument("--nb_epochs", type=int, default=100, help="Number of epochs")
- parser.add_argument("--model_dir", required=True, help="Path to save model")
+ parser.add_argument(
+ "--batch_size", type=int, default=8, help="Batch size"
+ )
+ parser.add_argument(
+ "--learning_rate", type=float, default=0.0001, help="Learning rate"
+ )
+ parser.add_argument(
+ "--nb_epochs", type=int, default=100, help="Number of epochs"
+ )
+ parser.add_argument(
+ "--model_dir", required=True, help="Path to save model"
+ )
return parser
def check_files_order(files1, files2):
"""
Here we check that the two input lists of str are correctly sorted.
- Except for the last, splits of files1[i] and files2[i] from the "_" character, must be equal.
+ Except for the last, splits of files1[i] and files2[i] from the "_"
+ character, must be equal.
:param files1: list of filenames (str)
:param files2: list of filenames (str)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
index 9299c9e0..84b8726e 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
@@ -2,27 +2,63 @@
This example shows how to use the otbtf python API to train a deep net from patches-images.
"""
from otbtf import DatasetFromPatchesImages
-from otbtf.examples.tensorflow_v2x.fcnn import helper
from otbtf.examples.tensorflow_v2x.fcnn import fcnn_model
+from otbtf.examples.tensorflow_v2x.fcnn import helper
parser = helper.base_parser()
-parser.add_argument("--train_xs", required=True, nargs="+", default=[],
- help="A list of patches-images for the XS image (training dataset)")
-parser.add_argument("--train_labels", required=True, nargs="+", default=[],
- help="A list of patches-images for the labels (training dataset)")
-parser.add_argument("--valid_xs", required=True, nargs="+", default=[],
- help="A list of patches-images for the XS image (validation dataset)")
-parser.add_argument("--valid_labels", required=True, nargs="+", default=[],
- help="A list of patches-images for the labels (validation dataset)")
-parser.add_argument("--test_xs", required=False, nargs="+", default=[],
- help="A list of patches-images for the XS image (test dataset)")
-parser.add_argument("--test_labels", required=False, nargs="+", default=[],
- help="A list of patches-images for the labels (test dataset)")
-
-
-def create_dataset(xs_filenames, labels_filenames, batch_size, targets_keys=[fcnn_model.TARGET_NAME]):
+parser.add_argument(
+ "--train_xs",
+ required=True,
+ nargs="+",
+ default=[],
+ help="A list of patches-images for the XS image (training dataset)"
+)
+parser.add_argument(
+ "--train_labels",
+ required=True,
+ nargs="+",
+ default=[],
+ help="A list of patches-images for the labels (training dataset)"
+)
+parser.add_argument(
+ "--valid_xs",
+ required=True,
+ nargs="+",
+ default=[],
+ help="A list of patches-images for the XS image (validation dataset)"
+)
+parser.add_argument(
+ "--valid_labels",
+ required=True,
+ nargs="+",
+ default=[],
+ help="A list of patches-images for the labels (validation dataset)"
+)
+parser.add_argument(
+ "--test_xs",
+ required=False,
+ nargs="+",
+ default=[],
+ help="A list of patches-images for the XS image (test dataset)"
+)
+parser.add_argument(
+ "--test_labels",
+ required=False,
+ nargs="+",
+ default=[],
+ help="A list of patches-images for the labels (test dataset)"
+)
+
+
+def create_dataset(
+ xs_filenames: list,
+ labels_filenames: list,
+ batch_size: int,
+ targets_keys: list = [fcnn_model.TARGET_NAME]
+):
"""
- Returns a TF dataset generated from an `otbtf.DatasetFromPatchesImages` instance
+ Returns a TF dataset generated from an `otbtf.DatasetFromPatchesImages`
+ instance
"""
# Sort patches and labels
xs_filenames.sort()
@@ -32,27 +68,44 @@ def create_dataset(xs_filenames, labels_filenames, batch_size, targets_keys=[fcn
helper.check_files_order(xs_filenames, labels_filenames)
# Create dataset from the filename dict
- # You can add the `use_streaming` option here, is you want to lower the memory budget.
- # However, this can slow down your process since the patches are read on-the-fly on the filesystem.
- # Good when one batch computation is slower than one batch gathering!
- # You can also use a custom `Iterator` of your own (default is `RandomIterator`). See `otbtf.dataset.Iterator`.
- ds = DatasetFromPatchesImages(filenames_dict={"input_xs_patches": xs_filenames, "labels_patches": labels_filenames})
-
- # We generate the TF dataset, and we use a preprocessing option to put the labels into one hot encoding (see the
- # `fcnn_model.dataset_preprocessing_fn` function). Also, we set the `target_keys` parameter to ask the dataset to
- # deliver samples in the form expected by keras, i.e. a tuple of dicts (inputs_dict, target_dict).
- tf_ds = ds.get_tf_dataset(batch_size=batch_size, preprocessing_fn=fcnn_model.dataset_preprocessing_fn,
- targets_keys=targets_keys)
+ # You can add the `use_streaming` option here, is you want to lower the
+ # memory budget. However, this can slow down your process since the
+ # patches are read on-the-fly on the filesystem. Good when one batch
+ # computation is slower than one batch gathering! You can also use a
+ # custom `Iterator` of your own (default is `RandomIterator`).
+ # See `otbtf.dataset.Iterator`.
+ ds = DatasetFromPatchesImages(
+ filenames_dict={
+ "input_xs_patches": xs_filenames,
+ "labels_patches": labels_filenames
+ }
+ )
+
+ # We generate the TF dataset, and we use a preprocessing option to put the
+ # labels in one hot encoding (see `fcnn_model.dataset_preprocessing_fn()`).
+ # Also, we set the `target_keys` parameter to ask the dataset to deliver
+ # samples in the form expected by keras, i.e. a tuple of dicts
+ # (inputs_dict, target_dict).
+ tf_ds = ds.get_tf_dataset(
+ batch_size=batch_size,
+ preprocessing_fn=fcnn_model.dataset_preprocessing_fn,
+ targets_keys=targets_keys
+ )
return tf_ds
def train(params):
# Create TF datasets
- ds_train = create_dataset(params.train_xs, params.train_labels, batch_size=params.batch_size)
- ds_valid = create_dataset(params.valid_xs, params.valid_labels, batch_size=params.batch_size)
- ds_test = create_dataset(params.test_xs, params.test_labels,
- batch_size=params.batch_size) if params.test_xs else None
+ ds_train = create_dataset(
+ params.train_xs, params.train_labels, batch_size=params.batch_size
+ )
+ ds_valid = create_dataset(
+ params.valid_xs, params.valid_labels, batch_size=params.batch_size
+ )
+ ds_test = create_dataset(
+ params.test_xs, params.test_labels, batch_size=params.batch_size
+ ) if params.test_xs else None
# Train the model
fcnn_model.train(params, ds_train, ds_valid, ds_test)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index 3fbfe472..3fa303c2 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -1,8 +1,10 @@
"""
-This example shows how to use the otbtf python API to train a deep net from TFRecords.
+This example shows how to use the otbtf python API to train a deep net from
+TFRecords.
-We expect that the files are stored in the following way, with m, n, and k denoting respectively
-the number of TFRecords files in the training, validation, and test datasets:
+We expect that the files are stored in the following way, with m, n, and k
+denoting respectively the number of TFRecords files in the training,
+validation, and test datasets:
/dataset_dir
/train
@@ -23,24 +25,31 @@ the number of TFRecords files in the training, validation, and test datasets:
"""
import os
+
from otbtf import TFRecords
-from otbtf.examples.tensorflow_v2x.fcnn import helper
from otbtf.examples.tensorflow_v2x.fcnn import fcnn_model
+from otbtf.examples.tensorflow_v2x.fcnn import helper
parser = helper.base_parser()
-parser.add_argument("--tfrecords_dir", required=True,
- help="Directory containing train, valid(, test) folders of TFRecords files")
+parser.add_argument(
+ "--tfrecords_dir",
+ required=True,
+ help="Directory containing train, valid(, test) folders of TFRecords files"
+)
def train(params):
- # Patches directories must contain 'train' and 'valid' dirs ('test' is not required)
+ # Patches directories must contain 'train' and 'valid' dirs ('test' is not
+ # required)
train_dir = os.path.join(params.tfrecords_dir, "train")
valid_dir = os.path.join(params.tfrecords_dir, "valid")
test_dir = os.path.join(params.tfrecords_dir, "test")
- kwargs = {"batch_size": params.batch_size,
- "target_keys": [fcnn_model.TARGET_NAME],
- "preprocessing_fn": fcnn_model.dataset_preprocessing_fn}
+ kwargs = {
+ "batch_size": params.batch_size,
+ "target_keys": [fcnn_model.TARGET_NAME],
+ "preprocessing_fn": fcnn_model.dataset_preprocessing_fn
+ }
# Training dataset. Must be shuffled
assert os.path.isdir(train_dir)
@@ -51,7 +60,9 @@ def train(params):
ds_valid = TFRecords(valid_dir).read(**kwargs)
# Test dataset (optional)
- ds_test = TFRecords(test_dir).read(**kwargs) if os.path.isdir(test_dir) else None
+ ds_test = TFRecords(test_dir).read(**kwargs) if os.path.isdir(
+ test_dir
+ ) else None
# Train the model
fcnn_model.train(params, ds_train, ds_valid, ds_test)
diff --git a/otbtf/model.py b/otbtf/model.py
index e11f2e44..3cbc0230 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -1,54 +1,101 @@
# -*- coding: utf-8 -*-
+# ==========================================================================
+#
+# Copyright 2018-2019 IRSTEA
+# Copyright 2020-2023 INRAE
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==========================================================================*/
""" Base class for models"""
+from typing import List, Dict, Any
import abc
import logging
-import tensorflow
+import tensorflow as tf
+TensorsDict = Dict[str, tf.keras.engine.keras_tensor.KerasTensor]
class ModelBase(abc.ABC):
"""
Base class for all models
"""
- def __init__(self, dataset_element_spec, input_keys=None, inference_cropping=None):
+ def __init__(
+ self,
+ dataset_element_spec: tf.TensorSpec,
+ input_keys: List[str] = None,
+ inference_cropping: List[int] = None
+ ):
"""
Model initializer, must be called **inside** the strategy.scope().
- :param dataset_element_spec: the dataset elements specification (shape, dtype, etc). Can be retrieved from the
- dataset instance simply with `ds.element_spec`
- :param input_keys: Optional. the keys of the inputs used in the model. If not specified, all inputs from the
- dataset will be considered.
- :param inference_cropping: list of number of pixels to be removed on each side of the output during inference.
- This list creates some additional outputs in the model, not used during training,
- only during inference. Default [16, 32, 64, 96, 128]
+ Args:
+ dataset_element_spec: the dataset elements specification (shape,
+ dtype, etc). Can be retrieved from a dataset instance `ds`
+ simply with `ds.element_spec`
+ input_keys: Optional keys of the inputs used in the model. If not
+ specified, all inputs from the dataset will be considered.
+ inference_cropping: list of number of pixels to be removed on each
+ side of the output for inference. Additional outputs are
+ created in the model, not used during training, only during
+ inference. Default [16, 32, 64, 96, 128]
+
"""
# Retrieve dataset inputs shapes
dataset_input_element_spec = dataset_element_spec[0]
- logging.info("Dataset input element spec: %s", dataset_input_element_spec)
+ logging.info(
+ "Dataset input element spec: %s", dataset_input_element_spec
+ )
if input_keys:
self.dataset_input_keys = input_keys
logging.info("Using input keys: %s", self.dataset_input_keys)
else:
self.dataset_input_keys = list(dataset_input_element_spec)
- logging.info("Found dataset input keys: %s", self.dataset_input_keys)
-
- self.inputs_shapes = {key: dataset_input_element_spec[key].shape[1:] for key in self.dataset_input_keys}
+ logging.info(
+ "Found dataset input keys: %s", self.dataset_input_keys
+ )
+
+ self.inputs_shapes = {
+ key: dataset_input_element_spec[key].shape[1:]
+ for key in self.dataset_input_keys
+ }
logging.info("Inputs shapes: %s", self.inputs_shapes)
# Setup cropping, normalization function
- self.inference_cropping = [16, 32, 64, 96, 128] if not inference_cropping else inference_cropping
+ self.inference_cropping = inference_cropping or [16, 32, 64, 96, 128]
logging.info("Inference cropping values: %s", self.inference_cropping)
# Create model
self.model = self.create_network()
- def __getattr__(self, name):
- """This method is called when the default attribute access fails. We choose to try to access the attribute of
- self.model. Thus, any method of keras.Model() can be used transparently, e.g. model.summary() or model.fit()"""
+ def __getattr__(self, name: str) -> Any:
+ """
+ This method is called when the default attribute access fails. We
+ choose to try to access the attribute of self.model. Thus, any method
+ of `keras.Model()` can be used transparently, e.g. `model.summary()`
+ or model.fit()
+
+ Args:
+ name: name of the attribute
+
+ Returns:
+ attribute
+
+ """
return getattr(self.model, name)
- def get_inputs(self):
+ def get_inputs(self) -> TensorsDict:
"""
This method returns the dict of keras.Input
"""
@@ -57,47 +104,71 @@ class ModelBase(abc.ABC):
for key in self.dataset_input_keys:
new_shape = list(self.inputs_shapes[key])
logging.info("Original shape for input %s: %s", key, new_shape)
- # Here we modify the x and y dims of >2D tensors to enable any image size at input
+ # Here we modify the x and y dims of >2D tensors to enable any
+ # image size at input
if len(new_shape) > 2:
new_shape[0] = None
new_shape[1] = None
- placeholder = tensorflow.keras.Input(shape=new_shape, name=key)
+ placeholder = tf.keras.Input(shape=new_shape, name=key)
logging.info("New shape for input %s: %s", key, new_shape)
model_inputs.update({key: placeholder})
return model_inputs
@abc.abstractmethod
- def get_outputs(self, normalized_inputs):
+ def get_outputs(self, normalized_inputs: TensorsDict) -> TensorsDict:
"""
Implementation of the model, from the normalized inputs.
- :param normalized_inputs: normalized inputs, as generated from `self.normalize_inputs()`
- :return: dict of model outputs
+ Params:
+ normalized_inputs: normalized inputs, as generated from
+ `self.normalize_inputs()`
+
+ Returns:
+ model outputs
+
"""
- raise NotImplementedError("This method has to be implemented. Here you code the model :)")
+ raise NotImplementedError(
+ "This method has to be implemented. Here you code the model :)"
+ )
- def normalize_inputs(self, inputs):
+ def normalize_inputs(self, inputs: TensorsDict) -> TensorsDict:
"""
Normalize the model inputs.
Takes the dict of inputs and returns a dict of normalized inputs.
- :param inputs: model inputs
- :return: a dict of normalized model inputs
+ Params:
+ inputs: model inputs
+
+ Returns:
+ a dict of normalized model inputs
+
"""
- logging.warning("normalize_input() undefined. No normalization of the model inputs will be performed. "
- "You can implement the function in your model class if you want.")
+ logging.warning(
+ "normalize_input() undefined. No normalization of the model "
+ "inputs will be performed. You can implement the function in your "
+ "model class if you want."
+ )
return inputs
- def postprocess_outputs(self, outputs, inputs=None, normalized_inputs=None):
+ def postprocess_outputs(
+ self,
+ outputs: TensorsDict,
+ inputs: TensorsDict = None,
+ normalized_inputs: TensorsDict = None
+ ) -> TensorsDict:
"""
Post-process the model outputs.
Takes the dicts of inputs and outputs, and returns a dict of post-processed outputs.
- The default implementation provides a set of cropped output tensors
+ The default implementation provides a set of cropped output tensors.
+
+ Params:
+ outputs: dict of model outputs
+ inputs: dict of model inputs (optional)
+ normalized_inputs: dict of normalized model inputs (optional)
+
+ Returns:
+ a dict of post-processed model outputs
- :param outputs: dict of model outputs
- :param inputs: dict of model inputs (optional)
- :param normalized_inputs: dict of normalized model inputs (optional)
- :return: a dict of post-processed model outputs
"""
# Add extra outputs for inference
@@ -105,20 +176,29 @@ class ModelBase(abc.ABC):
for out_key, out_tensor in outputs.items():
for crop in self.inference_cropping:
extra_output_key = cropped_tensor_name(out_key, crop)
- extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
- logging.info("Adding extra output for tensor %s with crop %s (%s)", out_key, crop, extra_output_name)
+ extra_output_name = cropped_tensor_name(
+ out_tensor._keras_history.layer.name, crop
+ )
+ logging.info(
+ "Adding extra output for tensor %s with crop %s (%s)",
+ out_key, crop, extra_output_name
+ )
cropped = out_tensor[:, crop:-crop, crop:-crop, :]
- identity = tensorflow.keras.layers.Activation('linear', name=extra_output_name)
+ identity = tf.keras.layers.Activation(
+ 'linear', name=extra_output_name
+ )
extra_outputs[extra_output_key] = identity(cropped)
return extra_outputs
- def create_network(self):
+ def create_network(self) -> tf.keras.Model:
"""
- This method returns the Keras model. This needs to be called **inside** the strategy.scope().
- Can be reimplemented depending on the needs.
+ This method returns the Keras model. This needs to be called
+ **inside** the strategy.scope(). Can be reimplemented depending on the
+ needs.
- :return: the keras model
+ Returns:
+ the keras model
"""
# Get the model inputs
@@ -134,12 +214,19 @@ class ModelBase(abc.ABC):
logging.info("Model outputs: %s", outputs)
# Post-processing for inference
- postprocessed_outputs = self.postprocess_outputs(outputs=outputs, inputs=inputs,
- normalized_inputs=normalized_inputs)
+ postprocessed_outputs = self.postprocess_outputs(
+ outputs=outputs,
+ inputs=inputs,
+ normalized_inputs=normalized_inputs
+ )
outputs.update(postprocessed_outputs)
# Return the keras model
- return tensorflow.keras.Model(inputs=inputs, outputs=outputs, name=self.__class__.__name__)
+ return tf.keras.Model(
+ inputs=inputs,
+ outputs=outputs,
+ name=self.__class__.__name__
+ )
def summary(self, strategy=None):
"""
@@ -150,29 +237,35 @@ class ModelBase(abc.ABC):
if not strategy or _is_chief(strategy):
self.model.summary(line_length=150)
- def plot(self, output_path, strategy=None, show_shapes=False):
+ def plot(self, output_path: str, strategy=None, show_shapes: bool = False):
"""
Enables to save a figure representing the architecture of the network.
- Needs pydot and graphviz to work (`pip install pydot` and https://graphviz.gitlab.io/download/)
+ Needs pydot and graphviz to work (`pip install pydot` and
+ https://graphviz.gitlab.io/download/)
- :param output_path: output path for the schema
- :param strategy: strategy
- :param show_shapes: annotate with shapes values (True or False)
+ Params:
+ output_path: output path for the schema
+ strategy: strategy
+ show_shapes: annotate with shapes values (True or False)
"""
assert self.model, "Plot() only works if create_network() has been called beforehand"
# When multiworker strategy, only plot if the worker is chief
if not strategy or _is_chief(strategy):
- tensorflow.keras.utils.plot_model(self.model, output_path, show_shapes=show_shapes)
+ tf.keras.utils.plot_model(self.model, output_path, show_shapes=show_shapes)
def _is_chief(strategy):
"""
Tell if the current worker is the chief.
- :param strategy: strategy
- :return: True if the current worker is the chief, False else
+ Params:
+ strategy: strategy
+
+ Returns:
+ True if the current worker is the chief, False else
+
"""
# Note: there are two possible `TF_CONFIG` configuration.
# 1) In addition to `worker` tasks, a `chief` task type is use;
@@ -185,17 +278,25 @@ def _is_chief(strategy):
# is added because it is effectively run with only a single worker.
if strategy.cluster_resolver: # this means MultiWorkerMirroredStrategy
- task_type, task_id = strategy.cluster_resolver.task_type, strategy.cluster_resolver.task_id
- return (task_type == 'chief') or (task_type == 'worker' and task_id == 0) or task_type is None
+ task_type = strategy.cluster_resolver.task_type
+ task_id = strategy.cluster_resolver.task_id
+ return (task_type == 'chief') \
+ or (task_type == 'worker' and task_id == 0) \
+ or task_type is None
# strategy with only one worker
return True
-def cropped_tensor_name(tensor_name, crop):
+def cropped_tensor_name(tensor_name: str, crop: int):
"""
A name for the padded tensor
- :param tensor_name: tensor name
- :param pad: pad value
- :return: name
+
+ Params:
+ tensor_name: tensor name
+ crop: cropping value
+
+ Returns:
+ name for the cropped tensor
+
"""
return "{}_crop{}".format(tensor_name, crop)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 15a23354..af0a0bd1 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -2,7 +2,7 @@
# ==========================================================================
#
# Copyright 2018-2019 IRSTEA
-# Copyright 2020-2022 INRAE
+# Copyright 2020-2023 INRAE
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,51 +18,76 @@
#
# ==========================================================================*/
"""
-The tfrecords module provides an implementation for the TFRecords files read/write
+The tfrecords module provides an implementation for the TFRecords files
+read/write
"""
import glob
import json
-import os
import logging
+import os
from functools import partial
+
import tensorflow as tf
from tqdm import tqdm
+import otbtf
+
class TFRecords:
"""
- This class allows to convert Dataset objects to TFRecords and to load them in dataset tensorflows format.
+ This class allows to convert Dataset objects to TFRecords and to load them
+ in dataset tensorflow format.
"""
- def __init__(self, path):
+ def __init__(self, path: str):
"""
- :param path: Can be a directory where TFRecords must be saved/loaded
+ Params:
+ path: Can be a directory where TFRecords must be saved/loaded
"""
self.dirpath = path
os.makedirs(self.dirpath, exist_ok=True)
- self.output_types_file = os.path.join(self.dirpath, "output_types.json")
- self.output_shapes_file = os.path.join(self.dirpath, "output_shapes.json")
- self.output_shapes = self.load(self.output_shapes_file) if os.path.exists(self.output_shapes_file) else None
- self.output_types = self.load(self.output_types_file) if os.path.exists(self.output_types_file) else None
+ self.output_types_file = os.path.join(
+ self.dirpath, "output_types.json"
+ )
+ self.output_shapes_file = os.path.join(
+ self.dirpath, "output_shapes.json"
+ )
+ self.output_shapes = self.load(self.output_shapes_file) \
+ if os.path.exists(self.output_shapes_file) else None
+ self.output_types = self.load(self.output_types_file) \
+ if os.path.exists(self.output_types_file) else None
@staticmethod
def _bytes_feature(value):
"""
Convert a value to a type compatible with tf.train.Example.
- :param value: value
- :return a bytes_list from a string / byte.
+
+ Params:
+ value: value
+
+ Returns:
+ a bytes_list from a string / byte.
"""
if isinstance(value, type(tf.constant(0))):
value = value.numpy() # BytesList won't unpack a string from an EagerTensor.
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
- def ds2tfrecord(self, dataset, n_samples_per_shard=100, drop_remainder=True):
+ def ds2tfrecord(
+ self,
+ dataset: otbtf.Dataset,
+ n_samples_per_shard: int = 100,
+ drop_remainder: bool = True
+ ):
"""
Convert and save samples from dataset object to tfrecord files.
- :param dataset: Dataset object to convert into a set of tfrecords
- :param n_samples_per_shard: Number of samples per shard
- :param drop_remainder: Whether additional samples should be dropped. Advisable if using multiworkers training.
- If True, all TFRecords will have `n_samples_per_shard` samples
+
+ Params:
+ dataset: Dataset object to convert into a set of tfrecords
+ n_samples_per_shard: Number of samples per shard
+ drop_remainder: Whether additional samples should be dropped.
+ Advisable if using multiworkers training. If True, all
+ TFRecords will have `n_samples_per_shard` samples
+
"""
logging.info("%s samples", dataset.size)
@@ -70,10 +95,16 @@ class TFRecords:
if not drop_remainder and dataset.size % n_samples_per_shard > 0:
nb_shards += 1
- output_shapes = {key: output_shape for key, output_shape in dataset.output_shapes.items()}
+ output_shapes = {
+ key: output_shape
+ for key, output_shape in dataset.output_shapes.items()
+ }
self.save(output_shapes, self.output_shapes_file)
- output_types = {key: output_type.name for key, output_type in dataset.output_types.items()}
+ output_types = {
+ key: output_type.name
+ for key, output_type in dataset.output_types.items()
+ }
self.save(output_types, self.output_types_file)
for i in tqdm(range(nb_shards)):
@@ -87,100 +118,165 @@ class TFRecords:
with tf.io.TFRecordWriter(filepath) as writer:
for _ in range(nb_sample):
sample = dataset.read_one_sample()
- serialized_sample = {name: tf.io.serialize_tensor(fea) for name, fea in sample.items()}
- features = {name: self._bytes_feature(serialized_tensor) for name, serialized_tensor in
- serialized_sample.items()}
+ serialized_sample = {
+ name: tf.io.serialize_tensor(fea)
+ for name, fea in sample.items()
+ }
+ features = {
+ name: self._bytes_feature(serialized_tensor)
+ for name, serialized_tensor in
+ serialized_sample.items()
+ }
tf_features = tf.train.Features(feature=features)
example = tf.train.Example(features=tf_features)
writer.write(example.SerializeToString())
@staticmethod
- def save(data, filepath):
+ def save(data: Dict[str, Any], filepath: str):
"""
Save data to JSON format.
- :param data: Data to save json format
- :param filepath: Output file name
+
+ Params:
+ data: Data to save json format
+ filepath: Output file name
+
"""
with open(filepath, 'w') as file:
json.dump(data, file, indent=4)
@staticmethod
- def load(filepath):
+ def load(filepath: str):
"""
Return data from JSON format.
- :param filepath: Input file name
+
+ Args:
+ filepath: Input file name
+
"""
with open(filepath, 'r') as file:
return json.load(file)
- def parse_tfrecord(self, example, target_keys, preprocessing_fn=None, **kwargs):
+ def parse_tfrecord(
+ self,
+ example: Any,
+ target_keys: List[str],
+ preprocessing_fn: Callable = None,
+ **kwargs
+ ):
"""
Parse example object to sample dict.
- :param example: Example object to parse
- :param target_keys: list of keys of the targets
- :param preprocessing_fn: Optional. A preprocessing function that process the input example
- :param kwargs: some keywords arguments for preprocessing_fn
+
+ Params:
+ example: Example object to parse
+ target_keys: list of keys of the targets
+ preprocessing_fn: Optional. A preprocessing function that process
+ the input example
+ kwargs: some keywords arguments for preprocessing_fn
+
"""
- read_features = {key: tf.io.FixedLenFeature([], dtype=tf.string) for key in self.output_types}
+ read_features = {
+ key: tf.io.FixedLenFeature([], dtype=tf.string)
+ for key in self.output_types
+ }
example_parsed = tf.io.parse_single_example(example, read_features)
# Tensor with right data type
for key, out_type in self.output_types.items():
- example_parsed[key] = tf.io.parse_tensor(example_parsed[key], out_type=out_type)
+ example_parsed[key] = tf.io.parse_tensor(
+ example_parsed[key],
+ out_type=out_type
+ )
# Ensure shape
for key, shape in self.output_shapes.items():
example_parsed[key] = tf.ensure_shape(example_parsed[key], shape)
# Preprocessing
- example_parsed_prep = preprocessing_fn(example_parsed, **kwargs) if preprocessing_fn else example_parsed
+ example_parsed_prep = preprocessing_fn(
+ example_parsed, **kwargs
+ ) if preprocessing_fn else example_parsed
# Differentiating inputs and targets
- input_parsed = {key: value for (key, value) in example_parsed_prep.items() if key not in target_keys}
- target_parsed = {key: value for (key, value) in example_parsed_prep.items() if key in target_keys}
+ input_parsed = {
+ key: value for (key, value) in example_parsed_prep.items()
+ if key not in target_keys
+ }
+ target_parsed = {
+ key: value for (key, value) in example_parsed_prep.items()
+ if key in target_keys
+ }
return input_parsed, target_parsed
- def read(self, batch_size, target_keys, n_workers=1, drop_remainder=True, shuffle_buffer_size=None,
- preprocessing_fn=None, shard_policy=tf.data.experimental.AutoShardPolicy.AUTO,
- prefetch_buffer_size=tf.data.experimental.AUTOTUNE,
- num_parallel_calls=tf.data.experimental.AUTOTUNE, **kwargs):
+ def read(
+ self,
+ batch_size: int,
+ target_keys: List[str],
+ n_workers: int = 1,
+ drop_remainder: bool = True,
+ shuffle_buffer_size: int = None,
+ preprocessing_fn: Callable = None,
+ shard_policy=tf.data.experimental.AutoShardPolicy.AUTO,
+ prefetch_buffer_size: int = tf.data.experimental.AUTOTUNE,
+ num_parallel_calls: int = tf.data.experimental.AUTOTUNE,
+ **kwargs
+ ):
"""
- Read all tfrecord files matching with pattern and convert data to tensorflow dataset.
- :param batch_size: Size of tensorflow batch
- :param target_keys: Keys of the target, e.g. ['s2_out']
- :param n_workers: number of workers, e.g. 4 if using 4 GPUs
- e.g. 12 if using 3 nodes of 4 GPUs
- :param drop_remainder: whether the last batch should be dropped in the case it has fewer than
- `batch_size` elements. True is advisable when training on multiworkers.
- False is advisable when evaluating metrics so that all samples are used
- :param shuffle_buffer_size: if None, shuffle is not used. Else, blocks of shuffle_buffer_size
- elements are shuffled using uniform random.
- :param preprocessing_fn: Optional. A preprocessing function that takes input examples as args and returns the
- preprocessed input examples. Typically, examples are composed of model inputs and
- targets. Model inputs and model targets must be computed accordingly to (1) what the
- model outputs and (2) what training loss needs. For instance, for a classification
- problem, the model will likely output the softmax, or activation neurons, for each
- class, and the cross entropy loss requires labels in one hot encoding. In this case,
- the preprocessing_fn has to transform the labels values (integer ranging from
- [0, n_classes]) in one hot encoding (vector of 0 and 1 of length n_classes). The
- preprocessing_fn should not implement such things as radiometric transformations from
- input to input_preprocessed, because those are performed inside the model itself
- (see `otbtf.ModelBase.normalize_inputs()`).
- :param shard_policy: sharding policy for the TFRecordDataset options
- :param prefetch_buffer_size: buffer size for the prefetch operation
- :param num_parallel_calls: number of parallel calls for the parsing + preprocessing step
- :param kwargs: some keywords arguments for preprocessing_fn
+ Read all tfrecord files matching with pattern and convert data to
+ tensorflow dataset.
+
+ Params:
+ batch_size: Size of tensorflow batch
+ target_keys: Keys of the target, e.g. ['s2_out']
+ n_workers: number of workers, e.g. 4 if using 4 GPUs, e.g. 12 if
+ using 3 nodes of 4 GPUs
+ drop_remainder: whether the last batch should be dropped in the
+ case it has fewer than `batch_size` elements. True is
+ advisable when training on multiworkers. False is advisable
+ when evaluating metrics so that all samples are used
+ shuffle_buffer_size: if None, shuffle is not used. Else, blocks of
+ shuffle_buffer_size elements are shuffled using uniform random.
+ preprocessing_fn: Optional. A preprocessing function that takes
+ input examples as args and returns the preprocessed input
+ examples. Typically, examples are composed of model inputs and
+ targets. Model inputs and model targets must be computed
+ accordingly to (1) what the model outputs and (2) what
+ training loss needs. For instance, for a classification
+ problem, the model will likely output the softmax, or
+ activation neurons, for each class, and the cross entropy loss
+ requires labels in one hot encoding. In this case, the
+ `preprocessing_fn` has to transform the labels values (integer
+ ranging from [0, n_classes]) in one hot encoding (vector of 0
+ and 1 of length n_classes). The `preprocessing_fn` should not
+ implement such things as radiometric transformations from
+ input to input_preprocessed, because those are performed
+ inside the model itself (see
+ `otbtf.ModelBase.normalize_inputs()`).
+ shard_policy: sharding policy for the TFRecord dataset options
+ prefetch_buffer_size: buffer size for the prefetch operation
+ num_parallel_calls: number of parallel calls for the parsing +
+ preprocessing step
+ kwargs: some keywords arguments for `preprocessing_fn`
+
"""
- for dic, file in zip([self.output_types, self.output_shapes],
- [self.output_types_file, self.output_shapes_file]):
+ for dic, file in zip([self.output_types,
+ self.output_shapes],
+ [self.output_types_file,
+ self.output_shapes_file]):
assert dic, f"The file {file} is missing!"
options = tf.data.Options()
if shuffle_buffer_size:
- options.experimental_deterministic = False # disable order, increase speed
- options.experimental_distribute.auto_shard_policy = shard_policy # for multiworker
- parse = partial(self.parse_tfrecord, target_keys=target_keys, preprocessing_fn=preprocessing_fn, **kwargs)
+ # disable order, increase speed
+ options.experimental_deterministic = False
+ # for multiworker
+ options.experimental_distribute.auto_shard_policy = shard_policy
+ parse = partial(
+ self.parse_tfrecord,
+ target_keys=target_keys,
+ preprocessing_fn=preprocessing_fn,
+ **kwargs
+ )
# 1/ num_parallel_reads useful ? I/O bottleneck of not ?
# 2/ num_parallel_calls=tf.data.experimental.AUTOTUNE useful ?
@@ -188,14 +284,22 @@ class TFRecords:
matching_files = glob.glob(tfrecords_pattern_path)
logging.info('Searching TFRecords in %s...', tfrecords_pattern_path)
logging.info('Number of matching TFRecords: %s', len(matching_files))
- matching_files = matching_files[:n_workers * (len(matching_files) // n_workers)] # files multiple of workers
+ matching_files = matching_files[:n_workers * (
+ len(matching_files) // n_workers)] # files multiple of workers
nb_matching_files = len(matching_files)
if nb_matching_files == 0:
- raise Exception(f"At least one worker has no TFRecord file in {tfrecords_pattern_path}. Please ensure that "
- "the number of TFRecord files is greater or equal than the number of workers!")
+ raise Exception(
+ "At least one worker has no TFRecord file in "
+ f"{tfrecords_pattern_path}. Please ensure that the number of "
+ "TFRecord files is greater or equal than the number of "
+ "workers!"
+ )
logging.info('Reducing number of records to : %s', nb_matching_files)
- dataset = tf.data.TFRecordDataset(matching_files) # , num_parallel_reads=2) # interleaves reads from xxx files
- dataset = dataset.with_options(options) # uses data as soon as it streams in, rather than in its original order
+ dataset = tf.data.TFRecordDataset(
+ matching_files
+ ) # , num_parallel_reads=2) # interleaves reads from xxx files
+ # uses data as soon as it streams in, rather than in its original order
+ dataset = dataset.with_options(options)
dataset = dataset.map(parse, num_parallel_calls=num_parallel_calls)
if shuffle_buffer_size:
dataset = dataset.shuffle(buffer_size=shuffle_buffer_size)
diff --git a/otbtf/utils.py b/otbtf/utils.py
index 069638a5..c2b33084 100644
--- a/otbtf/utils.py
+++ b/otbtf/utils.py
@@ -2,7 +2,7 @@
# ==========================================================================
#
# Copyright 2018-2019 IRSTEA
-# Copyright 2020-2022 INRAE
+# Copyright 2020-2023 INRAE
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -24,13 +24,17 @@ from osgeo import gdal
import numpy as np
-# ----------------------------------------------------- Helpers --------------------------------------------------------
+# --------------------------------- Helpers -----------------------------------
-def gdal_open(filename):
+def gdal_open(filename: str):
"""
Open a GDAL raster
- :param filename: raster file
- :return: a GDAL dataset instance
+
+ Params:
+ filename: raster file
+
+ Returns:
+ one GDAL dataset instance
"""
gdal_ds = gdal.Open(filename)
if not gdal_ds:
@@ -38,14 +42,25 @@ def gdal_open(filename):
return gdal_ds
-def read_as_np_arr(gdal_ds, as_patches=True, dtype=None):
+def read_as_np_arr(
+ gdal_ds,
+ as_patches: bool = True,
+ dtype: np.dtype = None
+) -> np.ndarray:
"""
Read a GDAL raster as numpy array
- :param gdal_ds: a GDAL dataset instance
- :param as_patches: if True, the returned numpy array has the following shape (n, psz_x, psz_x, nb_channels). If
- False, the shape is (1, psz_y, psz_x, nb_channels)
- :param dtype: if not None array dtype will be cast to given numpy data type (np.float32, np.uint16...)
- :return: Numpy array of dim 4
+
+ Params:
+ gdal_ds: a GDAL dataset instance
+ as_patches: if True, the returned numpy array has the following
+ shape (n, psz_x, psz_x, nb_channels). If False, the shape is (1,
+ psz_y, psz_x, nb_channels)
+ dtype: if not None array dtype will be cast to given numpy data type
+ (np.float32, np.uint16...)
+
+ Returns
+ Numpy array of dim 4
+
"""
buffer = gdal_ds.ReadAsArray()
size_x = gdal_ds.RasterXSize
--
GitLab
From a3b064326f9ec0a87a5a7c89de919109238a92ba Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 27 Mar 2023 21:44:03 +0200
Subject: [PATCH 02/72] DOC: wip
---
README.md | 112 ++----
app/otbPatchesSelection.cxx | 5 +-
doc/APPLICATIONS.md | 371 ------------------
doc/CREATEYOUROWN.md | 37 --
doc/api_model_generalities.md | 56 +++
doc/api_tutorial.md | 357 +++++++++++++++++
doc/app_inference.md | 113 ++++++
doc/app_overview.md | 46 +++
doc/app_sampling.md | 164 ++++++++
doc/app_training.md | 111 ++++++
doc/{HOWTOBUILD.md => build_from_sources.md} | 88 ++++-
doc/custom_theme/main.html | 5 +
doc/doc_requirements.txt | 7 +
doc/docker_build.md | 237 +++++++++++
doc/docker_troubleshooting.md | 177 +++++++++
doc/{DOCKERUSE.md => docker_use.md} | 223 ++---------
doc/gen_ref_pages.py | 25 ++
doc/images/modelbase.png | 3 +
doc/images/modelbase_1.png | 3 +
doc/images/modelbase_2.png | 3 +
doc/images/modelbase_3.png | 3 +
doc/images/modelbase_4.png | 3 +
doc/images/pipeline.png | 3 +
doc/index.md | 112 ++++++
mkdocs.yml | 80 ++++
otbtf/examples/__init__.py | 4 +
.../examples/tensorflow_v1x/__init__.py | 324 ++++++++++-----
.../create_savedmodel_ienco-m3_patchbased.py | 252 ++++++++----
otbtf/examples/tensorflow_v2x/__init__.py | 0
.../tensorflow_v2x/deterministic/__init__.py | 92 +++++
otbtf/examples/tensorflow_v2x/fcnn/README.md | 64 ---
.../examples/tensorflow_v2x/fcnn/__init__.py | 4 +
.../tensorflow_v2x/fcnn/fcnn_model.py | 20 +-
otbtf/examples/tensorflow_v2x/l2_norm.py | 25 --
.../examples/tensorflow_v2x/scalar_product.py | 27 --
otbtf/model.py | 4 +-
test/api_unittest.py | 109 +++--
37 files changed, 2239 insertions(+), 1030 deletions(-)
delete mode 100644 doc/APPLICATIONS.md
delete mode 100644 doc/CREATEYOUROWN.md
create mode 100644 doc/api_model_generalities.md
create mode 100644 doc/api_tutorial.md
create mode 100644 doc/app_inference.md
create mode 100644 doc/app_overview.md
create mode 100644 doc/app_sampling.md
create mode 100644 doc/app_training.md
rename doc/{HOWTOBUILD.md => build_from_sources.md} (69%)
create mode 100644 doc/custom_theme/main.html
create mode 100644 doc/doc_requirements.txt
create mode 100644 doc/docker_build.md
create mode 100644 doc/docker_troubleshooting.md
rename doc/{DOCKERUSE.md => docker_use.md} (66%)
create mode 100644 doc/gen_ref_pages.py
create mode 100644 doc/images/modelbase.png
create mode 100644 doc/images/modelbase_1.png
create mode 100644 doc/images/modelbase_2.png
create mode 100644 doc/images/modelbase_3.png
create mode 100644 doc/images/modelbase_4.png
create mode 100644 doc/images/pipeline.png
create mode 100644 doc/index.md
create mode 100644 mkdocs.yml
create mode 100644 otbtf/examples/__init__.py
rename doc/EXAMPLES.md => otbtf/examples/tensorflow_v1x/__init__.py (50%)
create mode 100644 otbtf/examples/tensorflow_v2x/__init__.py
create mode 100644 otbtf/examples/tensorflow_v2x/deterministic/__init__.py
delete mode 100644 otbtf/examples/tensorflow_v2x/fcnn/README.md
create mode 100644 otbtf/examples/tensorflow_v2x/fcnn/__init__.py
delete mode 100644 otbtf/examples/tensorflow_v2x/l2_norm.py
delete mode 100644 otbtf/examples/tensorflow_v2x/scalar_product.py
diff --git a/README.md b/README.md
index 5b8bd66a..64dc851d 100644
--- a/README.md
+++ b/README.md
@@ -1,82 +1,42 @@
-
-
# OTBTF: Orfeo ToolBox meets TensorFlow
-[](https://opensource.org/licenses/Apache-2.0)
-[](https://gitlab.irstea.fr/remi.cresson/otbtf/-/commits/develop)
-
-This remote module of the [Orfeo ToolBox](https://www.orfeo-toolbox.org) provides a generic, multi purpose deep learning framework, targeting remote sensing images processing.
-It contains a set of new process objects that internally invoke [Tensorflow](https://www.tensorflow.org/), and a bunch of user-oriented applications to perform deep learning with real-world remote sensing images.
-Applications can be used to build OTB pipelines from Python or C++ APIs.
-
-## Features
-
-### OTB Applications
-
-- Sample patches in remote sensing images with `PatchesExtraction`,
-- Model training, supporting save/restore/import operations (a model can be trained from scratch or fine-tuned) with `TensorflowModelTrain`,
-- Inference with support of OTB streaming mechanism with `TensorflowModelServe`. The streaming mechanism means (1) no limitation with images sizes, (2) inference can be used as a "lego" in any OTB pipeline (using C++ or Python APIs) and preserving streaming, (3) MPI support available (use multiple processing unit to generate one single output image)
-
-### Python
-
-The `otbtf` module targets python developers that want to train their own model from python with TensorFlow or Keras.
-It provides various classes for datasets and iterators to handle the _patches images_ generated from the `PatchesExtraction` OTB application.
-For instance, the `otbtf.DatasetFromPatchesImages` can be instantiated from a set of _patches images_
-and delivering samples as `tf.dataset` that can be used in your favorite TensorFlow pipelines, or convert your patches into TFRecords.
-The `otbtf.TFRecords` enables you train networks from TFRecords files, which is quite suited for
-distributed training. Read more in the [tutorial for keras](otbtf/examples/tensorflow_v2x/fcnn/README.md).
-
-`tricks.py` is here for backward compatibility with codes based on OTBTF 1.x and 2.x.
-
-## Examples
-
-Below are some screen captures of deep learning applications performed at large scale with OTBTF.
- - Landcover mapping (Spot-7 images --> Building map using semantic segmentation)
-
-
-
- - Super resolution (Sentinel-2 images upsampled with the [SR4RS software](https://github.com/remicres/sr4rs), which is based on OTBTF)
-
-
-
- - Sentinel-2 reconstruction with Sentinel-1 VV/VH with the [Decloud software](https://github.com/CNES/decloud), which is based on OTBTF
-
-
- -
- - Image to image translation (Spot-7 image --> Wikimedia Map using CGAN. So unnecessary but fun!)
-
-
-
-## How to install
-
-For now you have two options: either use the existing **docker image**, or build everything **from source**.
-
-### Docker
-
-Use the latest CPU or GPU-enabled image from dockerhub:
-```
-docker run mdl4eo/otbtf:3.4.0-cpu otbcli_PatchesExtraction -help
+<p align="center">
+<img src="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/logo.png" width="160px">
+<br>
+<a href="https://gitlab.irstea.fr/remi.cresson/otbtf/-/releases">
+<img src="https://gitlab.irstea.fr/remi.cresson/otbtf/-/badges/release.svg">
+</a>
+<a href="https://gitlab.irstea.fr/remi.cresson/otbtf/-/commits/master">
+<img src="https://gitlab.irstea.fr/remi.cresson/otbtf/badges/master/pipeline.svg">
+</a>
+<a href="LICENSE">
+<img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg">
+</a>
+</p>
+
+OTBTF is a remote module of the [Orfeo ToolBox](https://www.orfeo-toolbox.org).
+It provides a generic, multi purpose deep learning framework, targeting remote
+sensing images processing. It contains a set of new process objects for OTB
+that internally invoke [Tensorflow](https://www.tensorflow.org/), and new [OTB
+applications](#otb-applications) to perform deep learning with real-world
+remote sensing images. Applications can be used to build OTB pipelines from
+Python or C++ APIs. OTBTF also includes a [python API](#python-api) to build
+quickly Keras compliant models suited for remote sensing imagery, easy to
+train in distributed environments.
+
+## Documentation
+
+[otbtf.readthedocs.io](otbtf.readthedocs.io)
+
+## Use
+
+We recommend to use our latest GPU enabled docker images.
+
+```bash
+docker run --runtime=nvidia -ti mdl4eo/otbtf:3.4.0-gpu otbcli_PatchesExtraction
+docker run --runtime=nvidia -ti mdl4eo/otbtf:3.4.0-gpu python -c "import otbtf"
```
-Read more in the [docker use documentation](doc/DOCKERUSE.md).
-
-### Build from sources
-
-Read more in the [build from sources documentation](doc/HOWTOBUILD.md).
-
-## How to use
-
-- Reading [the applications documentation](doc/APPLICATIONS.md) will help, of course 😉
-- A small [tutorial](https://mdl4eo.irstea.fr/2019/01/04/an-introduction-to-deep-learning-on-remote-sensing-images-tutorial/) on MDL4EO's blog
-- in the `python` folder are provided some [ready-to-use deep networks, with documentation and scientific references](doc/EXAMPLES.md).
-- A [book](https://doi.org/10.1201/9781003020851): *Cresson, R. (2020). Deep Learning for Remote Sensing Images with Open Source Software. CRC Press.* Use QGIS, OTB and Tensorflow to perform various kind of deep learning sorcery on remote sensing images (patch-based classification for landcover mapping, semantic segmentation of buildings, optical image restoration from joint SAR/Optical time series).
-- Check [our repository](https://github.com/remicres/otbtf_tutorials_resources) containing stuff (data and models) to begin with with!
-- Finally, take a look in the `test` folder. You will find plenty of command lines for applications tests!
-
-## Contribute
-
-Every one can **contribute** to OTBTF. Just open a PR :)
-
## Cite
```
@@ -90,4 +50,4 @@ Every one can **contribute** to OTBTF. Just open a PR :)
year={2018},
publisher={IEEE}
}
-```
+```
\ No newline at end of file
diff --git a/app/otbPatchesSelection.cxx b/app/otbPatchesSelection.cxx
index 3437849b..a146bb6a 100644
--- a/app/otbPatchesSelection.cxx
+++ b/app/otbPatchesSelection.cxx
@@ -110,9 +110,10 @@ public:
// Documentation
SetName("PatchesSelection");
SetDescription("This application generate points sampled at regular interval over "
- "the input image region. The grid size and spacing can be configured.");
+ "the input image region. The selection strategy, grid size and step, "
+ " can be configured.");
SetDocLongDescription("This application produces a vector data containing "
- "a set of points centered on the patches lying in the valid regions of the input image. ");
+ "a set of points centered on the selected patches.");
SetDocAuthors("Remi Cresson");
diff --git a/doc/APPLICATIONS.md b/doc/APPLICATIONS.md
deleted file mode 100644
index 2ca5fd1f..00000000
--- a/doc/APPLICATIONS.md
+++ /dev/null
@@ -1,371 +0,0 @@
-# Description of applications
-
-This section introduces the new OTB applications provided in OTBTF.
-
-## Patches extraction
-
-The `PatchesExtraction` application performs the extraction of patches in images from a vector data containing points.
-Each point locates the **center** of the **central pixel** of the patch.
-For patches with even size of *N*, the **central pixel** corresponds to the pixel index *N/2+1* (index starting at 0).
-We denote one _input source_, either an input image, or a stack of input images that will be concatenated (they must have the same size).
-The user can set the `OTB_TF_NSOURCES` environment variable to select the number of _input sources_ that he wants.
-For example, for sampling a Time Series (TS) together with a single Very High Resolution image (VHR), two sources are required:
- - 1 input images list for time series,
- - 1 input image for the VHR.
-
-The sampled patches are extracted at each positions designed by the input vector data, only if a patch lies fully in all _input sources_ extents.
-For each _input source_, patches sizes must be provided.
-For each _input source_, the application export all sampled patches as a single multiband raster, stacked in rows.
-For instance, for *n* samples of size *16 x 16* from a *4* channels _input source_, the output image will be a raster of size *16 x 16n* with *4* channels.
-An optional output is an image of size *1 x n* containing the value of one specific field of the input vector data.
-Typically, the *class* field can be used to generate a dataset suitable for a model that performs pixel wise classification.
-
-
-
-```
-This application extracts patches in multiple input images. Change the OTB_TF_NSOURCES environment variable to set the number of sources.
-Parameters:
- -source1 <group> Parameters for source 1
-MISSING -source1.il <string list> Input image(s) 1 (mandatory)
-MISSING -source1.out <string> [pixel] Output patches for image 1 [pixel=uint8/uint16/int16/uint32/int32/float/double/cint16/cint32/cfloat/cdouble] (default value is float) (mandatory)
-MISSING -source1.patchsizex <int32> X patch size for image 1 (mandatory)
-MISSING -source1.patchsizey <int32> Y patch size for image 1 (mandatory)
- -source1.nodata <float> No-data value for image 1
-MISSING -vec <string> Positions of the samples (must be in the same projection as input image) (mandatory)
- -outlabels <string> [pixel] output labels [pixel=uint8/uint16/int16/uint32/int32/float/double/cint16/cint32/cfloat/cdouble] (default value is uint8) (optional, off by default)
-MISSING -field <string> field of class in the vector data (mandatory)
- -progress <boolean> Report progress
- -help <string list> Display long help (empty list), or help for given parameters keys
-
-Use -help param1 [... paramN] to see detailed documentation of those parameters.
-
-Examples:
-otbcli_PatchesExtraction -vec points.sqlite -source1.il $s2_list -source1.patchsizex 16 -source1.patchsizey 16 -field class -source1.out outpatches_16x16.tif -outlabels outlabels.tif
-```
-
-## Build your Tensorflow model <a name="buildmodel"></a>
-
-You can build models using the TensorFlow Python API as shown in the `./python/` directory.
-Models must be exported in **SavedModel** format.
-When using a model in OTBTF, the important thing is to know the following parameters related to the _placeholders_ (the inputs of your model) and _output tensors_ (the outputs of your model).
- - For each _input placeholder_:
- - Name
- - **Receptive field**
- - For each _output tensor_:
- - Name
- - **Expression field**
- - **Scale factor**
-
-
-
-The **scale factor** describes the physical change of spacing of the outputs, typically introduced in the model by non unitary strides in pooling or convolution operators.
-For each output, it is expressed relatively to one single input of the model called the _reference input source_.
-Additionally, the names of the _target nodes_ must be known (e.g. "optimizer").
-Also, the names of _user placeholders_, typically scalars placeholders that are used to control some parameters of the model, must be know (e.g. "dropout_rate").
-The **receptive field** corresponds to the input volume that "sees" the deep net.
-The **expression field** corresponds to the output volume that the deep net will create.
-
-## Train your Tensorflow model
-
-Here we assume that you have produced patches using the **PatchesExtraction** application, and that you have a **SavedModel** stored in a directory somewhere on your filesystem.
-The **TensorflowModelTrain** application performs the training, validation (against test dataset, and against validation dataset) providing the usual metrics that machine learning frameworks provide (confusion matrix, recall, precision, f-score, ...).
-You must provide the path of the **SavedModel** to the `model.dir` parameter.
-The `model.restorefrom` and `model.saveto` corresponds to the variables of the **SavedModel** used respectively for restoring and saving them.
-Set you _input sources_ for training (`training` parameter group) and for validation (`validation` parameter group): the evaluation is performed against training data, and optionally also against the validation data (only if you set `validation.mode` to "class").
-For each _input sources_, the patch size and the placeholder name must be provided.
-Regarding validation, if a different name is found in a particular _input source_ of the `validation` parameter group, the application knows that the _input source_ is not fed to the model at inference, but is used as reference to compute evaluation metrics of the validation dataset.
-Batch size (`training.batchsize`) and number of epochs (`training.epochs`) can be set.
-_User placeholders_ can be set separately for training (`training.userplaceholders`) and validation (`validation.userplaceholders`).
-The `validation.userplaceholders` can be useful if you have a model that behaves differently depending the given placeholder.
-Let's take the example of dropout: it's nice for training, but you have to disable it to use the model at inference time.
-Hence you will pass a placeholder with "dropout\_rate=0.3" for training and "dropout\_rate=0.0" for validation.
-Of course, one can train models from handmade python code: to import the patches images, a convenient method consist in reading patches images as numpy arrays using OTB applications (e.g. **ExtractROI**) or GDAL, then do a np.reshape to the dimensions wanted.
-
-
-
-```
-Train a multisource deep learning net using Tensorflow. Change the OTB_TF_NSOURCES environment variable to set the number of sources.
-Parameters:
- -model <group> Model parameters
-MISSING -model.dir <string> Tensorflow model_save directory (mandatory)
- -model.restorefrom <string> Restore model from path (optional, off by default)
- -model.saveto <string> Save model to path (optional, off by default)
- -training <group> Training parameters
- -training.batchsize <int32> Batch size (mandatory, default value is 100)
- -training.epochs <int32> Number of epochs (mandatory, default value is 100)
- -training.userplaceholders <string list> Additional single-valued placeholders for training. Supported types: int, float, bool. (optional, off by default)
-MISSING -training.targetnodes <string list> Names of the target nodes (mandatory)
- -training.outputtensors <string list> Names of the output tensors to display (optional, off by default)
- -training.usestreaming <boolean> Use the streaming through patches (slower but can process big dataset) (optional, off by default, default value is false)
- -training.source1 <group> Parameters for source #1 (training)
-MISSING -training.source1.il <string list> Input image (or list to stack) for source #1 (training) (mandatory)
-MISSING -training.source1.patchsizex <int32> Patch size (x) for source #1 (mandatory)
-MISSING -training.source1.patchsizey <int32> Patch size (y) for source #1 (mandatory)
-MISSING -training.source1.placeholder <string> Name of the input placeholder for source #1 (training) (mandatory)
- -training.source2 <group> Parameters for source #2 (training)
-MISSING -training.source2.il <string list> Input image (or list to stack) for source #2 (training) (mandatory)
-MISSING -training.source2.patchsizex <int32> Patch size (x) for source #2 (mandatory)
-MISSING -training.source2.patchsizey <int32> Patch size (y) for source #2 (mandatory)
-MISSING -training.source2.placeholder <string> Name of the input placeholder for source #2 (training) (mandatory)
- -validation <group> Validation parameters
- -validation.step <int32> Perform the validation every Nth epochs (mandatory, default value is 10)
- -validation.mode <string> Metrics to compute [none/class/rmse] (mandatory, default value is none)
- -validation.userplaceholders <string list> Additional single-valued placeholders for validation. Supported types: int, float, bool. (optional, off by default)
- -validation.usestreaming <boolean> Use the streaming through patches (slower but can process big dataset) (optional, off by default, default value is false)
- -validation.source1 <group> Parameters for source #1 (validation)
- -validation.source1.il <string list> Input image (or list to stack) for source #1 (validation) (mandatory)
- -validation.source1.name <string> Name of the input placeholder or output tensor for source #1 (validation) (mandatory)
- -validation.source2 <group> Parameters for source #2 (validation)
- -validation.source2.il <string list> Input image (or list to stack) for source #2 (validation) (mandatory)
- -validation.source2.name <string> Name of the input placeholder or output tensor for source #2 (validation) (mandatory)
- -progress <boolean> Report progress
- -help <string list> Display long help (empty list), or help for given parameters keys
-
-Use -help param1 [... paramN] to see detailed documentation of those parameters.
-
-Examples:
-otbcli_TensorflowModelTrain -source1.il spot6pms.tif -source1.placeholder x1 -source1.patchsizex 16 -source1.patchsizey 16 -source2.il labels.tif -source2.placeholder y1 -source2.patchsizex 1 -source2.patchsizex 1 -model.dir /tmp/my_saved_model/ -training.userplaceholders is_training=true dropout=0.2 -training.targetnodes optimizer -model.saveto /tmp/my_saved_model/variables/variables
-```
-
-As you can note, there is `$OTB_TF_NSOURCES` + 1 sources because we often need at least one more source for the reference data (e.g. terrain truth for land cover mapping).
-
-## Inference
-
-The **TensorflowModelServe** application performs the inference, it can be used to produce an output raster with the specified tensors.
-Thanks to the streaming mechanism, very large images can be produced.
-The application uses the `TensorflowModelFilter` and a `StreamingFilter` to force the streaming of output.
-This last can be optionally disabled by the user, if he prefers using the extended filenames to deal with chunk sizes.
-However, it's still very useful when the application is used in other composites applications, or just without extended filename magic.
-Some models can consume a lot of memory.
-In addition, the native tiling strategy of OTB consists in strips but this might not always the best.
-For Convolutional Neural Networks for instance, square tiles are more interesting because the padding required to perform the computation of one single strip of pixels induces to input a lot more pixels that to process the computation of one single tile of pixels.
-So, this application takes in input one or multiple _input sources_ (the number of _input sources_ can be changed by setting the `OTB_TF_NSOURCES` to the desired number) and produce one output of the specified tensors.
-The user is responsible of giving the **receptive field** and **name** of _input placeholders_, as well as the **expression field**, **scale factor** and **name** of _output tensors_.
-The first _input source_ (`source1.il`) corresponds to the _reference input source_.
-As explained [previously](#buildmodel), the **scale factor** provided for the _output tensors_ is related to this _reference input source_.
-The user can ask for multiple _output tensors_, that will be stack along the channel dimension of the output raster.
-However, if the sizes of those _output tensors_ are not consistent (e.g. a different number of (x,y) elements), an exception will be thrown.
-
-
-
-
-```
-Multisource deep learning classifier using TensorFlow. Change the OTB_TF_NSOURCES environment variable to set the number of sources.
-Parameters:
- -source1 <group> Parameters for source #1
-MISSING -source1.il <string list> Input image (or list to stack) for source #1 (mandatory)
-MISSING -source1.rfieldx <int32> Input receptive field (width) for source #1 (mandatory)
-MISSING -source1.rfieldy <int32> Input receptive field (height) for source #1 (mandatory)
-MISSING -source1.placeholder <string> Name of the input placeholder for source #1 (mandatory)
- -model <group> model parameters
-MISSING -model.dir <string> TensorFlow model_save directory (mandatory)
- -model.userplaceholders <string list> Additional single-valued placeholders. Supported types: int, float, bool. (optional, off by default)
- -model.fullyconv <boolean> Fully convolutional (optional, off by default, default value is false)
- -output <group> Output tensors parameters
- -output.spcscale <float> The output spacing scale, related to the first input (mandatory, default value is 1)
-MISSING -output.names <string list> Names of the output tensors (mandatory)
- -output.efieldx <int32> The output expression field (width) (mandatory, default value is 1)
- -output.efieldy <int32> The output expression field (height) (mandatory, default value is 1)
- -optim <group> This group of parameters allows optimization of processing time
- -optim.disabletiling <boolean> Disable tiling (optional, off by default, default value is false)
- -optim.tilesizex <int32> Tile width used to stream the filter output (mandatory, default value is 16)
- -optim.tilesizey <int32> Tile height used to stream the filter output (mandatory, default value is 16)
-MISSING -out <string> [pixel] output image [pixel=uint8/uint16/int16/uint32/int32/float/double/cint16/cint32/cfloat/cdouble] (default value is float) (mandatory)
- -progress <boolean> Report progress
- -help <string list> Display long help (empty list), or help for given parameters keys
-
-Use -help param1 [... paramN] to see detailed documentation of those parameters.
-
-Examples:
-otbcli_TensorflowModelServe -source1.il spot6pms.tif -source1.placeholder x1 -source1.rfieldx 16 -source1.rfieldy 16 -model.dir /tmp/my_saved_model/ -model.userplaceholders is_training=false dropout=0.0 -output.names out_predict1 out_proba1 -out "classif128tgt.tif?&streaming:type=tiled&streaming:sizemode=height&streaming:sizevalue=256"
-```
-
-## Composite applications for classification
-
-Who has never dreamed to use classic classifiers performing on deep learning features?
-This is possible thank to two new applications that uses the existing training/classification applications of OTB:
-
-**TrainClassifierFromDeepFeatures**: is a composite application that wire the **TensorflowModelServe** application output into the existing official **TrainImagesClassifier** application.
-
-```
-Train a classifier from deep net based features of an image and training vector data.
-Parameters:
- -source1 <group> Parameters for source 1
-MISSING -source1.il <string list> Input image (or list to stack) for source #1 (mandatory)
-MISSING -source1.rfieldx <int32> Input receptive field (width) for source #1 (mandatory)
-MISSING -source1.rfieldy <int32> Input receptive field (height) for source #1 (mandatory)
-MISSING -source1.placeholder <string> Name of the input placeholder for source #1 (mandatory)
- -model <group> Deep net inputs parameters
-MISSING -model.dir <string> TensorFlow model_save directory (mandatory)
- -model.userplaceholders <string list> Additional single-valued placeholders. Supported types: int, float, bool. (optional, off by default)
- -model.fullyconv <boolean> Fully convolutional (optional, off by default, default value is false)
- -output <group> Deep net outputs parameters
- -output.spcscale <float> The output spacing scale, related to the first input (mandatory, default value is 1)
-MISSING -output.names <string list> Names of the output tensors (mandatory)
- -output.efieldx <int32> The output expression field (width) (mandatory, default value is 1)
- -output.efieldy <int32> The output expression field (height) (mandatory, default value is 1)
- -optim <group> Processing time optimization
- -optim.disabletiling <boolean> Disable tiling (optional, off by default, default value is false)
- -optim.tilesizex <int32> Tile width used to stream the filter output (mandatory, default value is 16)
- -optim.tilesizey <int32> Tile height used to stream the filter output (mandatory, default value is 16)
- -ram <int32> Available RAM (Mb) (optional, off by default, default value is 128)
-MISSING -vd <string list> Vector data for training (mandatory)
- -valid <string list> Vector data for validation (optional, off by default)
-MISSING -out <string> Output classification model (mandatory)
- -confmatout <string> Output confusion matrix (optional, off by default)
- -sample <group> Sampling parameters
- -sample.mt <int32> Maximum training sample size per class (mandatory, default value is 1000)
- -sample.mv <int32> Maximum validation sample size per class (mandatory, default value is 1000)
- -sample.bm <int32> Bound sample number by minimum (mandatory, default value is 1)
- -sample.vtr <float> Training and validation sample ratio (mandatory, default value is 0.5)
- -sample.vfn <string> Field containing the class integer label for supervision (mandatory, no default value)
- -elev <group> Elevation parameters
- -elev.dem <string> DEM directory (optional, off by default)
- -elev.geoid <string> Geoid File (optional, off by default)
- -elev.default <float> Default elevation (mandatory, default value is 0)
- -classifier <string> Classifier parameters [libsvm/boost/dt/gbt/ann/bayes/rf/knn/sharkrf/sharkkm] (mandatory, default value is libsvm)
- -classifier.libsvm.k <string> SVM Kernel Type [linear/rbf/poly/sigmoid] (mandatory, default value is linear)
- -classifier.libsvm.m <string> SVM Model Type [csvc/nusvc/oneclass] (mandatory, default value is csvc)
- -classifier.libsvm.c <float> Cost parameter C (mandatory, default value is 1)
- -classifier.libsvm.nu <float> Cost parameter Nu (mandatory, default value is 0.5)
- -classifier.libsvm.opt <boolean> Parameters optimization (mandatory, default value is false)
- -classifier.libsvm.prob <boolean> Probability estimation (mandatory, default value is false)
- -classifier.boost.t <string> Boost Type [discrete/real/logit/gentle] (mandatory, default value is real)
- -classifier.boost.w <int32> Weak count (mandatory, default value is 100)
- -classifier.boost.r <float> Weight Trim Rate (mandatory, default value is 0.95)
- -classifier.boost.m <int32> Maximum depth of the tree (mandatory, default value is 1)
- -classifier.dt.max <int32> Maximum depth of the tree (mandatory, default value is 65535)
- -classifier.dt.min <int32> Minimum number of samples in each node (mandatory, default value is 10)
- -classifier.dt.ra <float> Termination criteria for regression tree (mandatory, default value is 0.01)
- -classifier.dt.cat <int32> Cluster possible values of a categorical variable into K <= cat clusters to find a suboptimal split (mandatory, default value is 10)
- -classifier.dt.f <int32> K-fold cross-validations (mandatory, default value is 10)
- -classifier.dt.r <boolean> Set Use1seRule flag to false (mandatory, default value is false)
- -classifier.dt.t <boolean> Set TruncatePrunedTree flag to false (mandatory, default value is false)
- -classifier.gbt.w <int32> Number of boosting algorithm iterations (mandatory, default value is 200)
- -classifier.gbt.s <float> Regularization parameter (mandatory, default value is 0.01)
- -classifier.gbt.p <float> Portion of the whole training set used for each algorithm iteration (mandatory, default value is 0.8)
- -classifier.gbt.max <int32> Maximum depth of the tree (mandatory, default value is 3)
- -classifier.ann.t <string> Train Method Type [back/reg] (mandatory, default value is reg)
- -classifier.ann.sizes <string list> Number of neurons in each intermediate layer (mandatory)
- -classifier.ann.f <string> Neuron activation function type [ident/sig/gau] (mandatory, default value is sig)
- -classifier.ann.a <float> Alpha parameter of the activation function (mandatory, default value is 1)
- -classifier.ann.b <float> Beta parameter of the activation function (mandatory, default value is 1)
- -classifier.ann.bpdw <float> Strength of the weight gradient term in the BACKPROP method (mandatory, default value is 0.1)
- -classifier.ann.bpms <float> Strength of the momentum term (the difference between weights on the 2 previous iterations) (mandatory, default value is 0.1)
- -classifier.ann.rdw <float> Initial value Delta_0 of update-values Delta_{ij} in RPROP method (mandatory, default value is 0.1)
- -classifier.ann.rdwm <float> Update-values lower limit Delta_{min} in RPROP method (mandatory, default value is 1e-07)
- -classifier.ann.term <string> Termination criteria [iter/eps/all] (mandatory, default value is all)
- -classifier.ann.eps <float> Epsilon value used in the Termination criteria (mandatory, default value is 0.01)
- -classifier.ann.iter <int32> Maximum number of iterations used in the Termination criteria (mandatory, default value is 1000)
- -classifier.rf.max <int32> Maximum depth of the tree (mandatory, default value is 5)
- -classifier.rf.min <int32> Minimum number of samples in each node (mandatory, default value is 10)
- -classifier.rf.ra <float> Termination Criteria for regression tree (mandatory, default value is 0)
- -classifier.rf.cat <int32> Cluster possible values of a categorical variable into K <= cat clusters to find a suboptimal split (mandatory, default value is 10)
- -classifier.rf.var <int32> Size of the randomly selected subset of features at each tree node (mandatory, default value is 0)
- -classifier.rf.nbtrees <int32> Maximum number of trees in the forest (mandatory, default value is 100)
- -classifier.rf.acc <float> Sufficient accuracy (OOB error) (mandatory, default value is 0.01)
- -classifier.knn.k <int32> Number of Neighbors (mandatory, default value is 32)
- -classifier.sharkrf.nbtrees <int32> Maximum number of trees in the forest (mandatory, default value is 100)
- -classifier.sharkrf.nodesize <int32> Min size of the node for a split (mandatory, default value is 25)
- -classifier.sharkrf.mtry <int32> Number of features tested at each node (mandatory, default value is 0)
- -classifier.sharkrf.oobr <float> Out of bound ratio (mandatory, default value is 0.66)
- -classifier.sharkkm.maxiter <int32> Maximum number of iteration for the kmeans algorithm. (mandatory, default value is 10)
- -classifier.sharkkm.k <int32> The number of class used for the kmeans algorithm. (mandatory, default value is 2)
- -rand <int32> User defined random seed (optional, off by default)
- -inxml <string> Load otb application from xml file (optional, off by default)
- -progress <boolean> Report progress
- -help <string list> Display long help (empty list), or help for given parameters keys
-
-Use -help param1 [... paramN] to see detailed documentation of those parameters.
-
-Examples:
-None
-```
-
-**ImageClassifierFromDeepFeatures** same approach with the official **ImageClassifier**.
-
-```
-Classify image using features from a deep net and an OTB machine learning classification model
-Parameters:
- -source1 <group> Parameters for source 1
-MISSING -source1.il <string list> Input image (or list to stack) for source #1 (mandatory)
-MISSING -source1.rfieldx <int32> Input receptive field (width) for source #1 (mandatory)
-MISSING -source1.rfieldy <int32> Input receptive field (height) for source #1 (mandatory)
-MISSING -source1.placeholder <string> Name of the input placeholder for source #1 (mandatory)
- -deepmodel <group> Deep net model parameters
-MISSING -deepmodel.dir <string> TensorFlow model_save directory (mandatory)
- -deepmodel.userplaceholders <string list> Additional single-valued placeholders. Supported types: int, float, bool. (optional, off by default)
- -deepmodel.fullyconv <boolean> Fully convolutional (optional, off by default, default value is false)
- -output <group> Deep net outputs parameters
- -output.spcscale <float> The output spacing scale, related to the first input (mandatory, default value is 1)
-MISSING -output.names <string list> Names of the output tensors (mandatory)
- -output.efieldx <int32> The output expression field (width) (mandatory, default value is 1)
- -output.efieldy <int32> The output expression field (height) (mandatory, default value is 1)
- -optim <group> This group of parameters allows optimization of processing time
- -optim.disabletiling <boolean> Disable tiling (optional, off by default, default value is false)
- -optim.tilesizex <int32> Tile width used to stream the filter output (mandatory, default value is 16)
- -optim.tilesizey <int32> Tile height used to stream the filter output (mandatory, default value is 16)
-MISSING -model <string> Model file (mandatory)
- -imstat <string> Statistics file (optional, off by default)
- -nodatalabel <int32> Label mask value (optional, off by default, default value is 0)
-MISSING -out <string> [pixel] Output image [pixel=uint8/uint16/int16/uint32/int32/float/double/cint16/cint32/cfloat/cdouble] (default value is uint8) (mandatory)
- -confmap <string> [pixel] Confidence map image [pixel=uint8/uint16/int16/uint32/int32/float/double/cint16/cint32/cfloat/cdouble] (default value is double) (optional, off by default)
- -ram <int32> Ram (optional, off by default, default value is 128)
- -inxml <string> Load otb application from xml file (optional, off by default)
- -progress <boolean> Report progress
- -help <string list> Display long help (empty list), or help for given parameters keys
-
-Use -help param1 [... paramN] to see detailed documentation of those parameters.
-
-Examples:
-None
-```
-
-Note that you can still set the `OTB_TF_NSOURCES` environment variable.
-
-# Basic example
-
-Below is a minimal example that presents the main steps to train a model, and perform the inference.
-
-## Sampling
-
-Here we will try to provide a simple example of doing a classification using a deep net that performs on one single VHR image.
-Our data set consists in one Spot-7 image, *spot7.tif*, and a training vector data, *terrain_truth.shp* that describes sparsely forest / non-forest polygons.
-First, we compute statistics of the vector data : how many points can we sample inside objects, and how many objects in each class.
-We use the **PolygonClassStatistics** application of OTB.
-```
-otbcli_PolygonClassStatistics -vec terrain_truth.shp -field class -in spot7.tif -out vec_stats.xml
-```
-Then, we will select some samples with the **SampleSelection** application of the existing machine learning framework of OTB.
-Since the terrain truth is sparse, we want to sample randomly points in polygons with the default strategy of the **SampleSelection** OTB application.
-```
-otbcli_SampleSelection -in spot7.tif -vec terrain_truth.shp -instats vec_stats.xml -field class -out points.shp
-```
-Now we extract the patches with the **PatchesExtraction** application.
-We want to produce one image of 16x16 patches, and one image for the corresponding labels.
-```
-otbcli_PatchesExtraction -source1.il spot7.tif -source1.patchsizex 16 -source1.patchsizey 16 -vec points.shp -field class -source1.out samp_labels.tif -outpatches samp_patches.tif
-```
-
-## Training
-
-Now we have two images for patches and labels.
-We can split them to distinguish test/validation groups (with the **ExtractROI** application for instance).
-But here, we will just perform some fine tuning of our model.
-The **SavedModel** is located in the `outmodel` directory.
-Our model is quite basic: it has two input placeholders, **x1** and **y1** respectively for input patches (with size 16x16) and input reference labels (with size 1x1).
-We named **prediction** the tensor that predict the labels and the optimizer that perform the stochastic gradient descent is an operator named **optimizer**.
-We perform the fine tuning and we export the new model variables directly in the _outmodel/variables_ folder, overwriting the existing variables of the model.
-We use the **TensorflowModelTrain** application to perform the training of this existing model.
-```
-otbcli_TensorflowModelTrain -model.dir /path/to/oursavedmodel -training.targetnodesnames optimizer -training.source1.il samp_patches.tif -training.source1.patchsizex 16 -training.source1.patchsizey 16 -training.source1.placeholder x1 -training.source2.il samp_labels.tif -training.source2.patchsizex 1 -training.source2.patchsizey 1 -training.source2.placeholder y1 -model.saveto /path/to/oursavedmodel/variables/variables
-```
-Note that we could also have performed validation in this step. In this case, the `validation.source2.placeholder` would be different than the `training.source2.placeholder`, and would be **prediction**. This way, the program know what is the target tensor to evaluate.
-
-## Inference
-
-After this step, we use the trained model to produce the entire map of forest over the whole Spot-7 image.
-For this, we use the **TensorflowModelServe** application to produce the **prediction** tensor output for the entire image.
-```
-otbcli_TensorflowModelServe -source1.il spot7.tif -source1.placeholder x1 -source1.rfieldx 16 -source1.rfieldy 16 -model.dir /path/to/oursavedmodel -output.names prediction -out map.tif uint8
-```
diff --git a/doc/CREATEYOUROWN.md b/doc/CREATEYOUROWN.md
deleted file mode 100644
index 4a22d4ef..00000000
--- a/doc/CREATEYOUROWN.md
+++ /dev/null
@@ -1,37 +0,0 @@
-# Create your own architecture
-
-This section gives a few tips to create your own models ready to be used in inference using OTBTF's `TensorflowModelServe` and `TensorflowModelTrain` applications.
-
-## Model inputs
-
-### Dimensions
-
-All networks must input **4D tensors**.
-- **dim 0** is for the batch dimension. It is used in the `TensorflowModelTrain` application during training, and in **patch-based mode** during inference: in this mode, `TensorflowModelServe` performs the inference of several patches simultaneously. In **fully-convolutional mode**, a single slice of the batch dimension is used.
-- **dim 1** and **2** are for the spatial dimensions,
-- **dim 3** is for the image channels. Even if your image have only 1 channel, you must set a shape value equals to 1 for the last dimension of the input placeholder.
-
-### Shapes
-
-For nets intended to work in **patch-based** mode, you can stick with a placeholder where you define your patch size explicitly in **dim 1** and **dim 2**.
-However, for nets intended to work in **fully-convolutional** mode, you must set `None` in **dim 1** and **dim 2** (before Tensorflow 2.X, it was possible to feed placeholders with a tensor of different size where the dims were defined, but no more after!).
-For instance, let consider an input raster with 4 spectral bands: the input shape of the model input would be like `[None, None, None, 4]` to work in fully-convolutional mode. By doing so, the use of input images of any size is enabled (`TensorflowModelServe` will automatically compute the input/output regions sizes to process, given the **receptive field** and **expression field** of your net).
-
-## Model outputs
-
-### Dimensions
-
-Supported tensors for the outputs must have **between 2 and 4 dimensions**.
-OTBTF always consider that **the size of the last dimension is the number of channels in the output**.
-For instance, you can have a model that outputs 8 channels with a tensor of shape `[None, 8]` or `[None, None, None, 8]`
-
-### Name your tensors and nodes
-
-Always name explicitly your models outputs. You will need the output tensor name for performing the inference with `TensoflowModelServe`. If you forget to name them, use the graph viewer in `tensorboard` to get the names.
-
-### Training
-
-If you want to enable your network training with the `TensorflowModelTrain` application, do not forget to name your optimizers/operators!
-You can build a single operator from multiple ones using the `tf.group` command, which also enable you to name your new operator.
-For sequential nodes trigger, you can build an operator that do what you want is the desired order using the `tf.control_dependancies` with TF <= 1.15.
-
diff --git a/doc/api_model_generalities.md b/doc/api_model_generalities.md
new file mode 100644
index 00000000..e0c0f42a
--- /dev/null
+++ b/doc/api_model_generalities.md
@@ -0,0 +1,56 @@
+# Model generalities
+
+This section gives a few tips to create your own models ready to be used in
+OTBTF.
+
+## Inputs dimensions
+
+All networks must input **4D tensors**.
+
+- **dim 0** is for the batch dimension. It is used in the
+`TensorflowModelTrain` application during training, and in
+**patch-based mode** during inference: in this mode, `TensorflowModelServe`
+performs the inference of several patches simultaneously. In
+**fully-convolutional mode**, a single slice of the batch dimension is used.
+- **dim 1** and **2** are for the spatial dimensions,
+- **dim 3** is for the image channels. Even if your image have only 1 channel,
+you must set a shape value equals to 1 for the last dimension of the input placeholder.
+
+## Inputs shapes
+
+For nets intended to work in **patch-based** mode, you can stick with a
+placeholder where you define your patch size explicitly in **dim 1** and
+**dim 2**. However, for nets intended to work in **fully-convolutional** mode,
+you must set `None` in **dim 1** and **dim 2** (before Tensorflow 2.X, it was
+possible to feed placeholders with a tensor of different size where the dims
+were defined). For instance, let consider an input raster with 4 spectral
+bands: the input shape of the model input would be like `[None, None, None, 4]`
+to work in fully-convolutional mode. By doing so, the use of input images of
+any size is enabled (`TensorflowModelServe` will automatically compute the
+input/output regions sizes to process, given the **receptive field** and
+**expression field** of your net).
+
+## Outputs dimensions
+
+Supported tensors for the outputs must have **between 2 and 4 dimensions**.
+OTBTF always consider that **the size of the last dimension is the number of
+channels in the output**.
+For instance, you can have a model that outputs 8 channels with a tensor of
+shape `[None, 8]` or `[None, None, None, 8]`
+
+## Outputs names
+
+Always name explicitly your models outputs. You will need the output tensor
+name for performing the inference with `TensoflowModelServe`. If you forget to
+name them, use the graph viewer in `tensorboard` to get the names.
+
+!!! note
+
+ If you want to enable your network training with the `TensorflowModelTrain`
+ application, you can use the Tensorflow API v1. In this case, do not forget
+ to name your optimizers/operators. You can build a single operator from
+ multiple ones using the `tf.group` command, which also enable you to name
+ your new operator. For sequential nodes trigger (e.g. GANs), you can build
+ an operator that do what you want is the desired order using
+ `tf.control_dependancies()`.
+
diff --git a/doc/api_tutorial.md b/doc/api_tutorial.md
new file mode 100644
index 00000000..b4832b7a
--- /dev/null
+++ b/doc/api_tutorial.md
@@ -0,0 +1,357 @@
+# Build and train deep learning models
+
+This section covers the use of the Python API.
+It shows how to build and train a small fully convolutional model from patches
+extracted in the images. The example show how a model can be trained (1) from
+**patches-images**, or (2) from **TFRecords** files.
+
+## Classes and files
+
+All files are located in *otbtf/examples/tensorflow_v2x*.
+- **fcnn_model.py** implements a small fully convolutional U-Net like model,
+called `FCNNModel`, with the preprocessing and normalization functions that
+inherit from `otbtf.BaseModel`
+- **train_from_patches-images.py** shows how to train the model from a list of
+patches-images
+- **train_from_tfrecords.py** shows how to train the model from TFRecords files
+- **create_tfrecords.py** shows how to convert patch-images into TFRecords
+files
+- **helper.py** contains a few helping functions
+
+## Datasets
+
+TensorFlow datasets are the most practical way to feed a network data during
+training steps.
+In particular, they are very useful to train models with data parallelism using
+multiple workers (i.e. multiple GPU devices).
+Since OTBTF 3, two kind of approaches are available to deliver the patches:
+- Create TF datasets from **patches-images**: the first approach implemented in
+OTBTF, relying on geospatial raster formats supported by GDAL. Patches are
+stacked in rows. patches-images are friendly because they can be visualized
+like any other image. However this approach is **not very optimized**, since it
+generates a lot of I/O and stresses the filesystem when iterating randomly over
+patches.
+- Create TF datasets from **TFRecords** files. The principle is that a number of
+patches are stored in TFRecords files (google protobuf serialized data). This
+approach provides the best performances, since it generates less I/Os since
+multiple patches are read simultaneously together. It is the recommended approach
+to work on high end gear. It requires an additional step of converting the
+patches-images into TFRecords files.
+
+### Patches-images based datasets
+
+**Patches-images** are generated from the `PatchesExtraction` application of OTBTF.
+They consist in extracted patches stacked in rows into geospatial rasters.
+The `otbtf.DatasetFromPatchesImages` provides access to **patches-images** as a
+TF dataset. It inherits from the `otbtf.Dataset` class, which can be a base class
+to develop other raster based datasets.
+The `use_streaming` option can be used to read the patches on-the-fly
+on the filesystem. However, this can cause I/O bottleneck when one training step
+is shorter that fetching one batch of data. Typically, this is very common with
+small networks trained over large amount of data using multiple GPUs, causing the
+filesystem read operation being the weak point (and the GPUs wait for the batches
+to be ready). The class offers other functionalities, for instance changing the
+iterator class with a custom one (can inherit from `otbtf.dataset.IteratorBase`)
+which is, by default, an `otbtf.dataset.RandomIterator`. This could enable to
+control how the patches are walked, from the multiple patches-images of the
+dataset.
+
+Suppose you have extracted some patches with the `PatchesExtraction`
+application with 2 sources:
+ - Source "xs": patches images *xs_1.tif*, ..., *xs_N.tif*
+ - Source "labels": patches images *labels_1.tif*, ..., *labels_N.tif*
+To create a dataset from this set of patches can be done with
+`otbtf.DatasetFromPatchesImages` as shown below.
+
+```python
+dataset = DatasetFromPatchesImages(
+ filenames_dict={
+ "input_xs_patches": ["xs_1.tif", ..., "xs_N.tif"],
+ "labels_patches": ["labels_1.tif", ..., "labels_N.tif"]
+ }
+)
+```
+
+Getting the Tensorflow dataset is done doing:
+
+```python
+tf_dataset = dataset.get_tf_dataset(
+ batch_size=8,
+ targets_keys=["predictions"]
+)
+```
+
+Here the `targets_keys` list contains all the keys of the target tensors.
+We will explain later why this has to be specified.
+
+You can also convert the dataset into TFRecords files:
+
+```python
+tf_dataset.to_tfrecords(output_dir="/tmp/")
+```
+
+TFRecords are the subject of the next section!
+
+### TFRecords batches datasets
+
+**TFRecord** based datasets are implemented in the `otbtf.tfrecords` module.
+They basically deliver patches from the TFRecords files, which can be created
+with the `to_tfrecords()` method of the `otbtf.Dataset` based classes.
+Depending on the filesystem characteristics and the computational cost of one
+training step, it can be good to select the number of samples per TFRecords file.
+Another tweak is the shuffling: since one TFRecord file contains multiple patches,
+the way TFRecords files are accessed (sometimes, we need them to be randomly
+accessed), and the way patches are accessed (within a buffer, of size set with
+the `shuffle_buffer_size`), is crucial.
+
+Creating TFRecords based datasets is super easy:
+
+```python
+dataset = TFRecords("/tmp")
+tf_dataset = dataset.read(
+ shuffle_buffer_size=1000,
+ batch_size=8,
+ target_keys=["predictions"]
+)
+```
+
+## Model
+
+Let's define the setting for our model:
+
+```python
+# Number of classes estimated by the model
+N_CLASSES = 2
+
+# Name of the input
+INPUT_NAME = "input_xs"
+
+# Name of the target output
+TARGET_NAME = "predictions"
+
+# Name (prefix) of the output we will use at inference time
+OUTPUT_SOFTMAX_NAME = "predictions_softmax_tensor"
+```
+
+Now we want to implement our model. Of course we can build from scratch
+a model based on `tf.keras.Model`, but we will see how OTBTF helps a lot with
+`otbtf.BaseModel`.
+Let's take a look to this schema:
+
+
+
+As we can see, we can distinguish two main functional blocks:
+
+- training
+- inference
+
+During training, we need to preprocess the samples generated by the dataset to
+feed the network and the loss computation, that will guide how weights will be
+updated. This **data preprocessing** is generally required to format the data
+in the format expected by the model.
+
+
+
+In our example, the terrain truth consists in labels which are integer values
+ranging from 0 to 1. However, the loss function that computes the cross
+entropy expects one hot encoding. The first thing to do is hence to transform
+the labels values into a one hot vector:
+
+```python
+def dataset_preprocessing_fn(examples: dict):
+ return {
+ INPUT_NAME: examples["input_xs_patches"],
+ TARGET_NAME: tf.one_hot(
+ tf.squeeze(tf.cast(examples["labels_patches"], tf.int32), axis=-1),
+ depth=N_CLASSES
+ )
+ }
+```
+
+The model is intended to work on real world images, which have often 16 bits
+signed integers as pixel values. The model has to normalize these values such
+as they fit the [0, 1] range before applying the convolutions. This is called
+**normalization**.
+
+
+
+This is the purpose of `normalize_inputs()`, which has to be implemented as
+model method. The method inputs a dictionary of tensors, and returns a
+dictionary of normalized tensors. The transformation is done multiplying the
+input by 0.0001, which guarantee that the 12-bits encoded Spot-7 image pixels
+is in the [0, 1] range. Also, we cast the input tensor, which is originally of
+type integer, to floating point.
+
+```python
+class FCNNModel(ModelBase):
+ def normalize_inputs(self, inputs: dict):
+ return {INPUT_NAME: tf.cast(inputs[INPUT_NAME], tf.float32) * 0.0001}
+```
+
+Then we implement the model itself in `FCNNModel.get_outputs()`. The model
+must return a dictionary of tensors. All keys of the target tensors must be in
+the returned dictionary (in our case: the *predictions* tensor). These target
+keys will be used later by the optimizer to perform the optimization of the
+loss.
+
+
+
+Our model is built with an encoder composed of 4 downscaling convolutional
+blocks, and its mirrored reversed decoder with skip connections between the
+layers of same scale. The last layer is a softmax layer that estimates the
+probability distribution for each class, and its output is used to perform the
+computation of the cross entropy loss with the terrain truth one hot encoded
+labels. Its name is *predictions* so that the loss crosses the terrain truth
+and the estimated values.
+
+```python
+...
+ def get_outputs(self, normalized_inputs: dict) -> dict:
+
+ def _conv(inp, depth, name):
+ conv_op = tf.keras.layers.Conv2D(
+ filters=depth,
+ kernel_size=3,
+ strides=2,
+ activation="relu",
+ padding="same",
+ name=name
+ )
+ return conv_op(inp)
+
+ def _tconv(inp, depth, name, activation="relu"):
+ tconv_op = tf.keras.layers.Conv2DTranspose(
+ filters=depth,
+ kernel_size=3,
+ strides=2,
+ activation=activation,
+ padding="same",
+ name=name
+ )
+ return tconv_op(inp)
+
+ out_conv1 = _conv(normalized_inputs[INPUT_NAME], 16, "conv1")
+ out_conv2 = _conv(out_conv1, 32, "conv2")
+ out_conv3 = _conv(out_conv2, 64, "conv3")
+ out_conv4 = _conv(out_conv3, 64, "conv4")
+ out_tconv1 = _tconv(out_conv4, 64, "tconv1") + out_conv3
+ out_tconv2 = _tconv(out_tconv1, 32, "tconv2") + out_conv2
+ out_tconv3 = _tconv(out_tconv2, 16, "tconv3") + out_conv1
+ out_tconv4 = _tconv(out_tconv3, N_CLASSES, "classifier", None)
+
+ softmax_op = tf.keras.layers.Softmax(name=OUTPUT_SOFTMAX_NAME)
+ predictions = softmax_op(out_tconv4)
+
+ return {TARGET_NAME: predictions}
+
+```
+
+Now our model is complete.
+
+## Training, validation, and test
+
+In the following, we will use the Keras API using the `model.compile()` then
+`model.fit()` instructions.
+
+First we declare the strategy used. Here we chose
+`tf.distribute.MirroredStrategy` which enable to use multiple GPUs on one
+computing resource.
+
+```python
+ strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
+```
+
+Then we instantiate, compile, and train the model within the `strategy` scope.
+
+First, we create an instance of our model:
+
+```python
+ with strategy.scope():
+ model = FCNNModel(dataset_element_spec=ds_train.element_spec)
+```
+
+!!! Info
+
+ As you can see, we just have provided `dataset_element_spec` to the model
+ initialization. If you take a look to
+ [`ModelBase.__init__()`](reference/otbtf/model.html#otbtf.model.ModelBase.__init__)
+ you can notice the `inference_cropping` parameter, with the default values
+ set to [16, 32, 64, 96, 128]. What is the use of these values?
+ Take another look in
+ [`ModelBase.postprocess_outputs()`](reference/otbtf/model.html#otbtf.model.ModelBase.postprocess_outputs).
+ The model will create an array of output, each one cropped to one value of
+ `inference_cropping`. These cropped output enable to avoid or lower the
+ magnitude of the blocking artifacts in convolutional models, caused by the
+ convolutions.
+
+In all the following, we are still inside the `strategy` scope.
+After the model is instantiated, we compile it using:
+
+- a `tf.keras.losses.CategoricalCrossentropy` loss, that will compute the
+categorical cross-entropy between the target labels (delivered from the
+pre-processed dataset) and the target output returned from `get_output()` of
+our model
+- an Adam optimizer,
+- Precision and Recall metrics (respectively `tf.keras.metrics.Precision` and
+`tf.keras.metrics.Recall`), that will be later computed over the validation
+dataset
+
+```python
+ model.compile(
+ loss=tf.keras.losses.CategoricalCrossentropy(),
+ optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),
+ metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()]
+ )
+```
+
+We can then train our model using Keras:
+
+```python
+ model.fit(ds_train, epochs=100, validation_data=ds_valid)
+```
+
+At the end of the training (here we just perform 100 epochs over the training
+dataset, then stop), we could perform some evaluation over an additional test
+dataset:
+
+```python
+ model.evaluate(ds_test, batch_size=64)
+```
+
+Finally we can save our model as a SavedModel:
+
+```python
+ model.save("/tmp/my_1st_savedmodel")
+```
+
+The whole code is available in the *otbtf/examples/tensorflow_v2x/fcnn*
+directory.
+
+## Inference
+
+We can use the exported SavedModel, located in */tmp/my_1st_savedmodel*, using
+either:
+- The OTB command line interface,
+- The OTB Python wrapper,
+- The PyOTB Python wrapper,
+- The OTB Graphical User Interface,
+- If we have configured QGIS with the descriptors of OTBTF applications, we
+also could have used QGIS for that.
+
+In the following, we focus only the CLI and python ways.
+
+### Command Line Interface
+
+```commandline
+otbcli_TensorflowModelServe \\
+-source1.il $DATADIR/fake_spot6.jp2 \\
+-source1.rfieldx 128 \\
+-source1.rfieldy 128 \\
+-source1.placeholder "input_xs" \\
+-model.dir /tmp/my_1st_savedmodel \\
+-model.fullyconv on \\
+-output.names "predictions_crop32" \\
+-output.efieldx 64 \\
+-output.efieldy 64 \\
+-out softmax.tif
+```
\ No newline at end of file
diff --git a/doc/app_inference.md b/doc/app_inference.md
new file mode 100644
index 00000000..9b1f8840
--- /dev/null
+++ b/doc/app_inference.md
@@ -0,0 +1,113 @@
+# Inference
+
+In OTBTF, the `TensorflowModelServe` performs the inference.
+The application can run models processing any kind or number of input images,
+as soon as they have geographical information and can be read with GDAL, which
+is the underlying library for IO in OTB.
+
+## Models information
+
+Models can be built using Tensorflow/Keras.
+They must be exported in **SavedModel** format.
+When using a model in OTBTF for inference, the following parameters must be
+known:
+
+- For each *input* (or *placeholder* for models built with Tensorflow API v1):
+ - Name
+ - Receptive field
+- For each *output tensor*:
+ - Name
+ - Expression field
+ - Scale factor
+
+
+
+The **scale factor** describes the physical change of spacing of the outputs,
+typically introduced in the model by non unitary strides in pooling or
+convolution operators.
+For each output, it is expressed relatively to one single input of the model
+called the *reference input source*.
+Additionally, the names of the *target nodes* must be known (e.g. optimizers
+for Tensorflow API v1).
+Also, the names of *user placeholders*, typically scalars inputs that are
+used to control some parameters of the model, must be know.
+The **receptive field** corresponds to the input volume that "sees" the deep
+net.
+The **expression field** corresponds to the output volume that the deep net
+will create.
+
+
+## TensorflowModelServe
+
+The **TensorflowModelServe** application performs the inference, it can be used
+to produce an output raster with the specified tensors.
+Thanks to the streaming mechanism, very large images can be produced.
+The application uses the `TensorflowModelFilter` and a `StreamingFilter` to
+force the streaming of output.
+This last can be optionally disabled by the user, if he prefers using the
+extended filenames to deal with chunk sizes.
+However, it's still very useful when the application is used in other
+composites applications, or just without extended filename magic.
+Some models can consume a lot of memory.
+In addition, the native tiling strategy of OTB consists in strips but this
+might not always the best.
+For Convolutional Neural Networks for instance, square tiles are more
+interesting because the padding required to perform the computation of one
+single strip of pixels induces to input a lot more pixels that to process the
+computation of one single tile of pixels.
+So, this application takes in input one or multiple _input sources_ (the number
+of _input sources_ can be changed by setting the `OTB_TF_NSOURCES` to the
+desired number) and produce one output of the specified tensors.
+The user is responsible of giving the **receptive field** and **name** of
+_input placeholders_, as well as the **expression field**, **scale factor** and
+**name** of _output tensors_.
+The first _input source_ (`source1.il`) corresponds to the _reference input
+source_.
+As explained, the **scale factor** provided for the
+_output tensors_ is related to this _reference input source_.
+The user can ask for multiple _output tensors_, that will be stack along the
+channel dimension of the output raster.
+However, if the sizes of those _output tensors_ are not consistent (e.g. a
+different number of (x,y) elements), an exception will be thrown.
+
+
+
+The application description can be displayed using:
+
+```commandline
+otbcli_TensorflowModelServe --help
+```
+
+## Composite applications for classification
+
+To use classic classifiers performing on a deep learning model features, one
+can use a traditional classifier generated from the
+**TrainClassifierFromDeepFeatures** application, in the
+**ImageClassifierFromDeepFeatures** application, which implements the same
+approach with the official OTB **ImageClassifier** application.
+
+The application description can be displayed using:
+
+```commandline
+otbcli_ImageClassifierFromDeepFeatures --help
+```
+
+Note that you can still set the `OTB_TF_NSOURCES` environment variable.
+
+# Example
+
+We assume that we have already followed the *training* section, and we start
+from the files generated at the end of the training.
+
+After this step, we use the trained model to produce the entire map of forest
+over the whole Spot-7 image.
+For this, we use the **TensorflowModelServe** application to produce the *
+*prediction** tensor output for the entire image.
+
+```commandLine
+otbcli_TensorflowModelServe -source1.il spot7.tif -source1.placeholder x1 \
+-source1.rfieldx 16 -source1.rfieldy 16 \
+-model.dir /path/to/oursavedmodel \
+-output.names prediction -out map.tif uint8
+```
+
diff --git a/doc/app_overview.md b/doc/app_overview.md
new file mode 100644
index 00000000..b782e781
--- /dev/null
+++ b/doc/app_overview.md
@@ -0,0 +1,46 @@
+# Applications overview
+
+## Applications in OTB
+
+In OTB, applications are processes working on geospatial images, with a
+standardized interface. This interface enables the applications to be fully
+interoperable, and operated from various ways: C++, python, command line
+interface. The cool thing is that most of the applications support the
+so-called *streaming* mechanism that enable to process very large images with
+a limited memory footprint. Thanks to the interface shared by the OTB
+applications, we can use them as functional bricks to build large pipelines,
+that are memory and computationally efficient.
+
+!!! Info
+
+ As any OTB application, the new applications provided by OTBTF can be used
+ in command line interface, C++, or python.
+ For the best experience in python, we recommend to use OTB applications
+ using the excellent
+ [PyOTB](https://pyotb.readthedocs.io/en/master/quickstart.html).
+
+## New applications
+
+Here are the new applications provided by OTBTF.
+
+- **TensorflowModelServe**: Inference on real world remote sensing products
+- **PatchesExtraction**: extract patches in images
+- **PatchesSelection**: patches selection from rasters
+- **LabelImageSampleSelection**: select patches from a label image
+- **DensePolygonClassStatistics**: fast terrain truth polygons statistics
+- **TensorflowModelTrain**: training/validation (educational purpose)
+- **TrainClassifierFromDeepFeatures**: train traditionnal classifiers that use
+ features from deep nets (educational/experimental)
+- **ImageClassifierFromDeepFeatures**: use traditionnal classifiers with
+ features from deep nets (educational/experimental)
+
+Typically, you could build a pipeline like that without coding a single
+image process, only by using existing OTB applications, and bringing your own
+Tensorflow model inside (with the `TensorflowModelServe` application).
+
+
+
+The entire pipeline would be fully streamable, with a minimal memory footprint.
+Also, it should be noted that most OTB applications are multithreaded and
+benefit from multiple cores. Read more about streaming in OTB
+[here](https://www.orfeo-toolbox.org/CookBook/C++/StreamingAndThreading.html).
\ No newline at end of file
diff --git a/doc/app_sampling.md b/doc/app_sampling.md
new file mode 100644
index 00000000..480ff5e0
--- /dev/null
+++ b/doc/app_sampling.md
@@ -0,0 +1,164 @@
+# Sampling applications
+
+OTBTF sampling applications are OTB applications that focus on the extraction
+of samples in the remote sensing images.
+
+Main OTBTF applications for sampling are:
+
+- [`PatchesSelection`](#patchesselection)
+- [`PatchesExtraction`](#patchesextraction)
+
+Other applications were written for experimental and educational purposes, but
+could still fill some needs sometimes:
+
+- [`DensePolygonClassStatistics`](#densepolygonclassstatistics)
+- [`LabelImageSampleSelection`](#labelimagesampleselection)
+
+## PatchesSelection
+
+This application generate points sampled at regular interval over the input
+image region. The selection strategy, patches grid size and step can be
+configured. The application produces a vector data containing a set of points
+centered on the patches after the selection process.
+
+The following strategies are implemented:
+
+- Split: the classic training/validation/testing samples split,
+- Chessboard: training/validation over the patches grid in a chessboard
+fashion
+- All: all patches are selected
+- Balanced: using an additional terrain truth labels map to select patches
+a random locations that try to balance the patches population distribution,
+based on the class value.
+
+The application description can be displayed using:
+
+```commandline
+otbcli_PatchesSelection --help
+```
+
+## PatchesExtraction.
+
+The `PatchesExtraction` application performs the extraction of patches in
+images from the following:
+
+- a vector data containing points (mandatory)
+- at least one imagery source (mandatory). To change the number of sources,
+set the environment variable `OTB_TF_NSOURCES`
+- One exiting field name of the vector data to identify the different points.
+
+Each point of the vector data locates the **center** of the **central pixel**
+of one patch.
+For each source, the following parameters can be set:
+
+- the patch size (x and y): for patches with even size *N*, the **central
+pixel** corresponds to the pixel index *N/2+1* (index starting at 0).
+- a no-data value: If any pixel value inside the patch is equal to the
+provided value, the patch is rejected.
+- an output file name for the *patches image* that the application exports at
+the end of the sampling. Patches are stacked in rows and exported as common
+raster files supported by GDAL, without any geographical information.
+
+### Example with 2 sources
+
+We denote one _input source_, either an input image, or a stack of input images
+that will be concatenated (they must have the same size).
+The user can set the `OTB_TF_NSOURCES` environment variable to select the
+number of _input sources_ that he wants.
+For example, for sampling a Time Series (TS) together with a single Very High
+Resolution image (VHR), two sources are required:
+
+- 1 input images list for time series,
+- 1 input image for the VHR.
+
+The sampled patches are extracted at each position designed by the input
+vector data, only if a patch lies fully in all _input sources_ extents.
+For each _input source_, patches sizes must be provided.
+For each _input source_, the application export all sampled patches as a single
+multiband raster, stacked in rows.
+For instance, for *n* samples of size *16 x 16* from a *4* channels _input
+source_, the output image will be a raster of size *16 x 16n* with *4*
+channels.
+An optional output is an image of size *1 x n* containing the value of one
+specific field of the input vector data.
+Typically, the *class* field can be used to generate a dataset suitable for a
+model that performs pixel wise classification.
+
+
+
+The application description can be displayed using:
+
+```commandline
+otbcli_PatchesExtraction --help
+```
+
+
+## DensePolygonClassStatistics
+
+This application is a clone of the [PolygonClassStatistics](https://www.orfeo-toolbox.org/CookBook/Applications/app_PolygonClassStatistics.html)
+modified to use rasterization instead of vector based approach, making it
+faster.
+
+The application description can be displayed using:
+
+```commandline
+otbcli_DensePolygonClassStatistics --help
+```
+
+## LabelImageSampleSelection
+
+This application extracts points from an input label image. This application
+is like "SampleSelection", but uses an input label image, rather than an input
+vector data. It produces a vector data containing a set of points centered on
+the pixels of the input label image. The user can control the number of
+points. The default strategy consists in producing the same number of points
+in each class. If one class has a smaller number of points than requested,
+this one is adjusted.
+
+The application description can be displayed using:
+
+```commandline
+otbcli_LabelImageSampleSelection --help
+```
+
+## Example
+
+Below is a minimal example that presents some steps to sample patches from a
+sparse annotated vector data as terrain truth.
+Let's consider that our data set consists in one Spot-7 image, *spot7.tif*,
+and a training vector data, *terrain_truth.shp* that describes sparsely
+forest / non-forest polygons.
+
+First, we compute statistics of the vector data : how many points can we sample
+inside objects, and how many objects in each class.
+We use the **PolygonClassStatistics** application of OTB.
+
+```commandline
+otbcli_PolygonClassStatistics -vec terrain_truth.shp -field class \
+-in spot7.tif -out vec_stats.xml
+```
+
+Then, we will select some samples with the **SampleSelection** application of
+the existing machine learning framework of OTB.
+Since the terrain truth is sparse, we want to sample randomly points in
+polygons with the default strategy of the **SampleSelection** OTB application.
+
+```
+otbcli_SampleSelection -in spot7.tif -vec terrain_truth.shp \
+-instats vec_stats.xml -field class -out points.shp
+```
+
+Now we extract the patches with the **PatchesExtraction** application.
+We want to produce one image of 16x16 patches, and one image for the
+corresponding labels.
+
+```
+otbcli_PatchesExtraction -source1.il spot7.tif \
+-source1.patchsizex 16 -source1.patchsizey 16 \
+-vec points.shp -field class -source1.out samp_labels.tif \
+-outpatches samp_patches.tif
+```
+
+Now we can use the generated *samp_patches.tif* and *samp_labels.tif* in the
+`TensorflowModelTrain` application, or using the python API to build and train
+models with Keras.
\ No newline at end of file
diff --git a/doc/app_training.md b/doc/app_training.md
new file mode 100644
index 00000000..dd9fd2c8
--- /dev/null
+++ b/doc/app_training.md
@@ -0,0 +1,111 @@
+!!! Warning
+
+ This section is for educational purposes. No coding skills are required,
+ and it's easy to train an existing model built with the Tensorflow API
+ v1. To have a full control over the model implementation and training
+ process, the Tensorflow API v2 with Keras is the way to go.
+
+# TensorflowModelTrain
+
+Here we assume that you have produced patches using the **PatchesExtraction**
+application, and that you have a **SavedModel** stored in a directory somewhere
+on your filesystem.
+The **TensorflowModelTrain** application performs the training, validation (
+against test dataset, and against validation dataset) providing the usual
+metrics that machine learning frameworks provide (confusion matrix, recall,
+precision, f-score, ...).
+You must provide the path of the **SavedModel** to the `model.dir` parameter.
+The `model.restorefrom` and `model.saveto` corresponds to the variables of the
+**SavedModel** used respectively for restoring and saving them.
+Set you _input sources_ for training (`training` parameter group) and for
+validation (`validation` parameter group): the evaluation is performed against
+training data, and optionally also against the validation data (only if you
+set `validation.mode` to "class").
+For each _input sources_, the patch size and the placeholder name must be
+provided.
+Regarding validation, if a different name is found in a particular _input
+source_ of the `validation` parameter group, the application knows that the
+_input source_ is not fed to the model at inference, but is used as reference
+to compute evaluation metrics of the validation dataset.
+Batch size (`training.batchsize`) and number of epochs (`training.epochs`) can
+be set.
+_User placeholders_ can be set separately for
+training (`training.userplaceholders`) and
+validation (`validation.userplaceholders`).
+The `validation.userplaceholders` can be useful if you have a model that
+behaves differently depending the given placeholder.
+Let's take the example of dropout: it's nice for training, but you have to
+disable it to use the model at inference time.
+Hence you will pass a placeholder with "dropout\_rate=0.3" for training and "
+dropout\_rate=0.0" for validation.
+Of course, one can train models from handmade python code: to import the
+patches images, a convenient method consist in reading patches images as numpy
+arrays using OTB applications (e.g. **ExtractROI**) or GDAL, then do a
+np.reshape to the dimensions wanted.
+
+
+
+The application description can be displayed using:
+
+```commandLine
+otbcli_TensorflowModelTrain --help
+```
+
+As you can note, there is `$OTB_TF_NSOURCES` + 1 sources because we often need
+at least one more source for the reference data (e.g. terrain truth for land
+cover mapping).
+
+## Composite applications for classification
+
+Who has never dreamed to use classic classifiers performing on deep learning
+features?
+This is possible thank to two new applications that uses the existing
+training/classification applications of OTB:
+
+**TrainClassifierFromDeepFeatures**: is a composite application that wire the *
+*TensorflowModelServe** application output into the existing official *
+*TrainImagesClassifier** application.
+
+The application description can be displayed using:
+
+```commandLine
+otbcli_TrainClassifierFromDeepFeatures --help
+```
+
+# Example
+
+We assume that we have already followed the *sampling* section, and we start
+from the files generated at the end of the patches extraction.
+
+Now we have two images for patches and labels.
+We can split them to distinguish test/validation groups (with the **ExtractROI**
+ application for instance).
+But here, we will just perform some fine tuning of our model.
+The **SavedModel** is located in the `outmodel` directory.
+Our model is quite basic: it has two input placeholders, **x1** and **y1**
+respectively for input patches (with size 16x16) and input reference labels (
+with size 1x1).
+We named **prediction** the tensor that predict the labels and the optimizer
+that perform the stochastic gradient descent is an operator named **optimizer
+**.
+We perform the fine tuning and we export the new model variables directly in
+the _outmodel/variables_ folder, overwriting the existing variables of the
+model.
+We use the **TensorflowModelTrain** application to perform the training of this
+existing model.
+
+```
+otbcli_TensorflowModelTrain -model.dir /path/to/oursavedmodel \
+-training.targetnodesnames optimizer -training.source1.il samp_patches.tif \
+-training.source1.patchsizex 16 -training.source1.patchsizey 16 \
+-training.source1.placeholder x1 -training.source2.il samp_labels.tif \
+-training.source2.patchsizex 1 -training.source2.patchsizey 1 \
+-training.source2.placeholder y1 \
+-model.saveto /path/to/oursavedmodel/variables/variables
+```
+
+Note that we could also have performed validation in this step. In this case,
+the `validation.source2.placeholder` would be different than
+the `training.source2.placeholder`, and would be **prediction**. This way, the
+program know what is the target tensor to evaluate.
+
diff --git a/doc/HOWTOBUILD.md b/doc/build_from_sources.md
similarity index 69%
rename from doc/HOWTOBUILD.md
rename to doc/build_from_sources.md
index 82d3b115..4189db1b 100644
--- a/doc/HOWTOBUILD.md
+++ b/doc/build_from_sources.md
@@ -1,9 +1,20 @@
# How to build OTBTF from sources
-This remote module has been tested successfully on Ubuntu 18 with last CUDA drivers, TensorFlow r2.1 and OTB 7.1.0.
+Building OTBTF from sources is the hard way.
+These instructions explain how to build on Ubuntu 18 with last CUDA
+drivers, TensorFlow r2.1 and OTB 7.1.0.
+
+!!! Warning
+
+ This section is no longer maintained.
+ You can take a look in the `Dockerfile` to take notes how it's done with
+ up-to-date ubuntu versions.
## Build OTB
-First, **build the *release-7.1* branch of OTB from sources**. You can check the [OTB documentation](https://www.orfeo-toolbox.org/SoftwareGuide/SoftwareGuidech2.html) which details all the steps. It is quite easy thank to the SuperBuild, a cmake script that automates the build.
+
+First, **build the *release-7.1* branch of OTB from sources**. You can check
+the [OTB documentation](https://www.orfeo-toolbox.org/SoftwareGuide/SoftwareGuidech2.html)
+which details all the steps. It is quite easy thank to the SuperBuild, a cmake script that automates the build.
Create a folder for OTB, clone sources, configure OTB SuperBuild, and build it.
@@ -12,7 +23,19 @@ Install required packages:
```
sudo apt-get update
sudo apt-get upgrade
-sudo apt-get install sudo ca-certificates curl make cmake g++ gcc git libtool swig xvfb wget autoconf automake pkg-config zip zlib1g-dev unzip freeglut3-dev libboost-date-time-dev libboost-filesystem-dev libboost-graph-dev libboost-program-options-dev libboost-system-dev libboost-thread-dev libcurl4-gnutls-dev libexpat1-dev libfftw3-dev libgdal-dev libgeotiff-dev libglew-dev libglfw3-dev libgsl-dev libinsighttoolkit4-dev libkml-dev libmuparser-dev libmuparserx-dev libopencv-core-dev libopencv-ml-dev libopenthreads-dev libossim-dev libpng-dev libqt5opengl5-dev libqwt-qt5-dev libsvm-dev libtinyxml-dev qtbase5-dev qttools5-dev default-jdk python3-pip python3.6-dev python3.6-gdal python3-setuptools libxmu-dev libxi-dev qttools5-dev-tools bison software-properties-common dirmngr apt-transport-https lsb-release gdal-bin
+sudo apt-get install sudo ca-certificates curl make cmake g++ gcc git \
+ libtool swig xvfb wget autoconf automake pkg-config zip zlib1g-dev \
+ unzip freeglut3-dev libboost-date-time-dev libboost-filesystem-dev \
+ libboost-graph-dev libboost-program-options-dev libboost-system-dev \
+ libboost-thread-dev libcurl4-gnutls-dev libexpat1-dev libfftw3-dev \
+ libgdal-dev libgeotiff-dev libglew-dev libglfw3-dev libgsl-dev \
+ libinsighttoolkit4-dev libkml-dev libmuparser-dev libmuparserx-dev \
+ libopencv-core-dev libopencv-ml-dev libopenthreads-dev libossim-dev \
+ libpng-dev libqt5opengl5-dev libqwt-qt5-dev libsvm-dev libtinyxml-dev \
+ qtbase5-dev qttools5-dev default-jdk python3-pip python3.6-dev \
+ python3.6-gdal python3-setuptools libxmu-dev libxi-dev \
+ qttools5-dev-tools bison software-properties-common dirmngr \
+ apt-transport-https lsb-release gdal-bin
```
Build OTB from sources:
@@ -23,7 +46,8 @@ sudo chown $USER /work
mkdir /work/otb
cd /work/otb
mkdir build
-git clone -b release-7.1 https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git OTB
+git clone -b release-7.1 \
+ https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git OTB
cd build
```
@@ -36,7 +60,16 @@ ccmake /work/otb/OTB/SuperBuild
If you don't know how to configure options, you can use the following:
```
-cmake /work/otb/OTB/SuperBuild -DUSE_SYSTEM_BOOST=ON -DUSE_SYSTEM_CURL=ON -DUSE_SYSTEM_EXPAT=ON -DUSE_SYSTEM_FFTW=ON -DUSE_SYSTEM_FREETYPE=ON -DUSE_SYSTEM_GDAL=ON -DUSE_SYSTEM_GEOS=ON -DUSE_SYSTEM_GEOTIFF=ON -DUSE_SYSTEM_GLEW=ON -DUSE_SYSTEM_GLFW=ON -DUSE_SYSTEM_GLUT=ON -DUSE_SYSTEM_GSL=ON -DUSE_SYSTEM_ITK=ON -DUSE_SYSTEM_LIBKML=ON -DUSE_SYSTEM_LIBSVM=ON -DUSE_SYSTEM_MUPARSER=ON -DUSE_SYSTEM_MUPARSERX=ON -DUSE_SYSTEM_OPENCV=ON -DUSE_SYSTEM_OPENTHREADS=ON -DUSE_SYSTEM_OSSIM=ON -DUSE_SYSTEM_PNG=ON -DUSE_SYSTEM_QT5=ON -DUSE_SYSTEM_QWT=ON -DUSE_SYSTEM_TINYXML=ON -DUSE_SYSTEM_ZLIB=ON -DUSE_SYSTEM_SWIG=OFF -DOTB_WRAP_PYTHON=OFF
+cmake /work/otb/OTB/SuperBuild -DUSE_SYSTEM_BOOST=ON -DUSE_SYSTEM_CURL=ON \
+ -DUSE_SYSTEM_EXPAT=ON -DUSE_SYSTEM_FFTW=ON -DUSE_SYSTEM_FREETYPE=ON \
+ -DUSE_SYSTEM_GDAL=ON -DUSE_SYSTEM_GEOS=ON -DUSE_SYSTEM_GEOTIFF=ON \
+ -DUSE_SYSTEM_GLEW=ON -DUSE_SYSTEM_GLFW=ON -DUSE_SYSTEM_GLUT=ON \
+ -DUSE_SYSTEM_GSL=ON -DUSE_SYSTEM_ITK=ON -DUSE_SYSTEM_LIBKML=ON \
+ -DUSE_SYSTEM_LIBSVM=ON -DUSE_SYSTEM_MUPARSER=ON \
+ -DUSE_SYSTEM_MUPARSERX=ON -DUSE_SYSTEM_OPENCV=ON \
+ -DUSE_SYSTEM_OPENTHREADS=ON -DUSE_SYSTEM_OSSIM=ON -DUSE_SYSTEM_PNG=ON \
+ -DUSE_SYSTEM_QT5=ON -DUSE_SYSTEM_QWT=ON -DUSE_SYSTEM_TINYXML=ON \
+ -DUSE_SYSTEM_ZLIB=ON -DUSE_SYSTEM_SWIG=OFF -DOTB_WRAP_PYTHON=OFF
```
Then you can build OTB:
@@ -45,7 +78,10 @@ make -j $(grep -c ^processor /proc/cpuinfo)
```
## Build TensorFlow with shared libraries
-During this step, you have to **build Tensorflow from source** except if you want to use only the sampling applications of OTBTensorflow (in this case, skip this section).
+
+During this step, you have to **build Tensorflow from source** except if you
+want to use only the sampling applications of OTBTensorflow (in this case,
+skip this section).
### Bazel
First, install Bazel.
@@ -56,7 +92,9 @@ chmod +x bazel-0.29.1-installer-linux-x86_64.sh
export PATH="$PATH:$HOME/bin"
```
-If you fail to install properly Bazel, you can read the beginning of [the instructions](https://www.tensorflow.org/install/install_sources) that present alternative methods for this.
+If you fail to install properly Bazel, you can read the beginning of
+[the instructions](https://www.tensorflow.org/install/install_sources) that
+present alternative methods for this.
### Required packages
There is a few required packages that you need to install:
@@ -65,7 +103,9 @@ sudo python3 -m pip install --upgrade pip
sudo python3 -m pip install pip six numpy wheel mock keras future setuptools
```
-For a pure python3 install, you might need to workaround a bazel bug the following way:
+For a pure python3 install, you might need to workaround a bazel bug the
+following way:
+
```
sudo ln -s /usr/bin/python3 /usr/bin/python
```
@@ -80,24 +120,36 @@ cd /work/tf
git clone https://github.com/tensorflow/tensorflow.git
```
-Now configure the project. If you have CUDA and other NVIDIA stuff installed in your system, remember that you have to tell the script that it is in `/usr/` (no symlink required!). If you have CPU-only hardware, building Intel MKL is a good choice since it provides a significant speedup in computations.
+Now configure the project. If you have CUDA and other NVIDIA stuff installed
+in your system, remember that you have to tell the script that it is in
+`/usr/` (no symlink required!). If you have CPU-only hardware, building Intel
+MKL is a good choice since it provides a significant speedup in computations.
```
cd tensorflow
./configure
```
-Then, you have to build TensorFlow with the instructions sets supported by your CPU (For instance here is AVX, AVX2, FMA, SSE4.1, SSE4.2 that play fine on a modern intel CPU). You have to tell Bazel to build:
+Then, you have to build TensorFlow with the instructions sets supported by
+your CPU (For instance here is AVX, AVX2, FMA, SSE4.1, SSE4.2 that play fine
+on a modern intel CPU). You have to tell Bazel to build:
1. The TensorFlow python pip package
2. The libtensorflow_cc.so library
3. The libtensorflow_framework.so library
```
-bazel build -c opt --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-mfpmath=both --copt=-msse4.1 --copt=-msse4.2 //tensorflow:libtensorflow_framework.so //tensorflow:libtensorflow_cc.so //tensorflow:libtensorflow.so //tensorflow/tools/pip_package:build_pip_package --noincompatible_do_not_split_linking_cmdline
+bazel build -c opt --copt=-mavx --copt=-mavx2 --copt=-mfma \
+ --copt=-mfpmath=both --copt=-msse4.1 --copt=-msse4.2 \
+ //tensorflow:libtensorflow_framework.so \
+ //tensorflow:libtensorflow_cc.so //tensorflow:libtensorflow.so \
+ //tensorflow/tools/pip_package:build_pip_package \
+ --noincompatible_do_not_split_linking_cmdline
```
-*You might fail this step (e.g. missing packages). In this case, it's recommended to clear the bazel cache, using something like `rm $HOME/.cache/bazel/* -rf` before configuring and building everything!*
+*You might fail this step (e.g. missing packages). In this case, it's
+recommended to clear the bazel cache, using something like
+`rm $HOME/.cache/bazel/* -rf` before configuring and building everything!*
### Pip package
Build and deploy the pip package.
@@ -143,17 +195,21 @@ cp -r bazel-tensorflow/external/eigen_archive/Eigen /work/tf/installdir/include
cp -r tensorflow/lite/tools/make/downloads/absl/absl /work/tf/installdir/include
```
-Now you have a working copy of TensorFlow located in `/work/tf/installdir` that is ready to use in external C++ cmake projects :)
+Now you have a working copy of TensorFlow located in `/work/tf/installdir`
+that is ready to use in external C++ cmake projects :)
## Build the OTBTF remote module
Finally, we can build the OTBTF module.
-Clone the repository inside the OTB sources directory for remote modules: `/work/otb/OTB/Modules/Remote/`.
+Clone the repository inside the OTB sources directory for remote modules:
+`/work/otb/OTB/Modules/Remote/`.
Re configure OTB with cmake of ccmake, and set the following variables
- **Module_OTBTensorflow** to **ON**
- - **OTB_USE_TENSORFLOW** to **ON** (if you set to OFF, you will have only the sampling applications)
+ - **OTB_USE_TENSORFLOW** to **ON** (if you set to OFF, you will have only
+ the sampling applications)
- **TENSORFLOW_CC_LIB** to `/work/tf/installdir/lib/libtensorflow_cc.so`
- - **TENSORFLOW_FRAMEWORK_LIB** to `/work/tf/installdir/lib/libtensorflow_framework.so`
+ - **TENSORFLOW_FRAMEWORK_LIB** to
+ `/work/tf/installdir/lib/libtensorflow_framework.so`
- **tensorflow_include_dir** to `/work/tf/installdir/include`
Re build and re install OTB.
diff --git a/doc/custom_theme/main.html b/doc/custom_theme/main.html
new file mode 100644
index 00000000..d9b3e888
--- /dev/null
+++ b/doc/custom_theme/main.html
@@ -0,0 +1,5 @@
+{% extends "base.html" %}
+
+{% block extrahead %}
+<meta name="google-site-verification" content="lKzwW-jjP466yU5s9vwERWKm0DHBXWmH1sIN2sas7sQ" />
+{% endblock %}
\ No newline at end of file
diff --git a/doc/doc_requirements.txt b/doc/doc_requirements.txt
new file mode 100644
index 00000000..4f596924
--- /dev/null
+++ b/doc/doc_requirements.txt
@@ -0,0 +1,7 @@
+mkdocstrings
+mkdocstrings[crystal,python]
+mkdocs-material
+mkdocs-gen-files
+mkdocs-section-index
+mkdocs-literate-nav
+mkdocs-mermaid2-plugin
diff --git a/doc/docker_build.md b/doc/docker_build.md
new file mode 100644
index 00000000..61abc4c1
--- /dev/null
+++ b/doc/docker_build.md
@@ -0,0 +1,237 @@
+# Build your own docker images
+
+Docker build has to be called from the root of the repository (i.e. `docker
+build .` or `bash tools/docker/multibuild.sh`).
+You can build a custom image using `--build-arg` and several config files :
+
+- **Ubuntu** : `BASE_IMG` should accept any version, for additional packages
+see *tools/docker/build-deps-cli.txt* and *tools/docker/build-deps-gui.txt*.
+- **TensorFlow** : `TF` arg for the git branch or tag + *build-env-tf.sh* and
+BZL_* arguments for the build configuration. `ZIP_TF_BIN` allows you to save
+compiled binaries if you want to install it elsewhere.
+- **OrfeoToolBox** : `OTB` arg for the git branch or tag +
+*tools/docker/build-flags-otb.txt* to edit cmake flags. Set `KEEP_SRC_OTB` in
+order to preserve OTB git directory.
+
+### Base images
+
+```bash
+UBUNTU=20.04 # or 16.04, 18.04
+CUDA=11.2.2 # or 10.1, 10.2, 11.0.3
+CUDNN=8 # or 7
+IMG=ubuntu:$UBUNTU
+GPU_IMG=nvidia/cuda:$CUDA-cudnn$CUDNN-devel-ubuntu$UBUNTU
+```
+
+### Default arguments
+
+```bash
+BASE_IMG # mandatory
+CPU_RATIO=1
+GUI=false
+NUMPY_SPEC="==1.19.*"
+TF=v2.8.0
+OTB=8.1.0
+BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package"
+BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
+BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
+ZIP_TF_BIN=false
+KEEP_SRC_OTB=false
+SUDO=true
+
+# NumPy version requirement :
+# TF < 2.4 : "numpy<1.19.0,>=1.16.0"
+# TF >= 2.4 : "numpy==1.19.*"
+# TF >= 2.8 : "numpy==1.22.*"
+```
+
+### Bazel remote cache daemon
+
+If you just need to rebuild with different GUI or KEEP_SRC arguments, or may
+be a different branch of OTB, bazel cache will help you to rebuild everything
+except TF, even if the docker cache was purged (after `docker
+[system|builder] prune`).
+In order to recycle the cache, bazel config and TF git tag should be exactly
+the same, any change in [build-env-tf.sh](build-env-tf.sh) and `--build-arg`
+(if related to bazel env, cuda, mkl, xla...) may result in a fresh new build.
+
+Start a cache daemon - here with max 20GB but 10GB should be enough to save 2
+TF builds (GPU and CPU):
+
+```bash
+mkdir -p $HOME/.cache/bazel-remote
+docker run --detach -u 1000:1000 -v $HOME/.cache/bazel-remote:/data \
+ -p 9090:8080 buchgr/bazel-remote-cache --max_size=20
+```
+
+Then just add ` --network='host'` to the docker build command, or connect
+bazel to a remote server - see 'BZL_OPTIONS'.
+The other way of docker is a virtual bridge, but you'll need to edit the IP
+address.
+
+## Images build examples
+
+```bash
+# Build for CPU using default Dockerfiles args (without AWS, HDFS or GCP
+# support)
+docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:20.04 .
+
+# Clear bazel config var (deactivate default optimizations and unset
+# noaws/nogcp/nohdfs)
+docker build --network='host' -t otbtf:cpu \
+ --build-arg BASE_IMG=ubuntu:20.04 \
+ --build-arg BZL_CONFIGS= .
+
+# Enable MKL
+MKL_CONFIG="--config=nogcp --config=noaws --config=nohdfs --config=opt --config=mkl"
+docker build --network='host' -t otbtf:cpu-mkl \
+ --build-arg BZL_CONFIGS="$MKL_CONFIG" \
+ --build-arg BASE_IMG=ubuntu:20.04 .
+
+# Build for GPU (if you're building for your system only you should edit
+# CUDA_COMPUTE_CAPABILITIES in build-env-tf.sh)
+docker build --network='host' -t otbtf:gpu \
+ --build-arg BASE_IMG=nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 .
+
+# Build latest TF and OTB, set git branches/tags to clone
+docker build --network='host' -t otbtf:gpu-dev \
+ --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 \
+ --build-arg KEEP_SRC_OTB=true \
+ --build-arg TF=nightly \
+ --build-arg OTB=develop .
+
+# Build old release (TF-2.1)
+docker build --network='host' -t otbtf:oldstable-gpu \
+ --build-arg BASE_IMG=nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 \
+ --build-arg TF=r2.1 \
+ --build-arg NUMPY_SPEC="<1.19" \
+ --build-arg BAZEL_OPTIONS="--noincompatible_do_not_split_linking_cmdline --verbose_failures --remote_cache=http://localhost:9090" .
+# You could edit the Dockerfile in order to clone an old branch of the repo
+# instead of copying files from the build context
+```
+
+### Build for another machine and save TF compiled files
+
+Example with TF 2.5
+
+```bash
+# Use same ubuntu and CUDA version than your target machine, beware of CC
+# optimization and CPU compatibility (set env variable CC_OPT_FLAGS and avoid
+# "-march=native" if your Docker's CPU is optimized with AVX2/AVX512 but your
+# target CPU isn't)
+docker build --network='host' -t otbtf:custom \
+ --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 \
+ --build-arg TF=v2.5.0 \
+ --build-arg ZIP_TF_BIN=true .
+# Retrieve zip file
+docker run -v $HOME:/home/otbuser/volume otbtf:custom \
+ cp /opt/otbtf/tf-v2.5.0.zip /home/otbuser/volume
+
+# Target machine shell
+cd $HOME
+unzip tf-v2.5.0.zip
+sudo mkdir -p /opt/tensorflow/lib
+sudo mv tf-v2.5.0/libtensorflow_cc* /opt/tensorflow/lib
+# You may need to create a virtualenv, here TF and dependencies are installed
+# next to user's pip packages
+pip3 install -U pip wheel mock six future deprecated "numpy==1.19.*"
+pip3 install --no-deps keras_applications keras_preprocessing
+pip3 install tf-v2.5.0/tensorflow-2.5.0-cp38-cp38-linux_x86_64.whl
+
+TF_WHEEL_DIR="$HOME/.local/lib/python3.8/site-packages/tensorflow"
+# If you installed the wheel as regular user, with root pip it should be in
+# /usr/local/lib/python3.*, or in your virtualenv lib/ directory
+mv tf-v2.5.0/tag_constants.h $TF_WHEEL_DIR/include/tensorflow/cc/saved_model/
+# Then recompile OTB with OTBTF using libraries in /opt/tensorflow/lib and
+# instructions in build_from_sources.md.
+cmake $OTB_GIT \
+ -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \
+ -DTENSORFLOW_CC_LIB=/opt/tensorflow/lib/libtensorflow_cc.so.2 \
+ -Dtensorflow_include_dir=$TF_WHEEL_DIR/include \
+ -DTENSORFLOW_FRAMEWORK_LIB=$TF_WHEEL_DIR/libtensorflow_framework.so.2 \
+&& make install -j
+```
+
+### Debug build
+
+If you fail to build, you can log into the last layer and check CMake logs.
+Run `docker images`, find the latest layer ID and run a tmp container
+(`docker run -it d60496d9612e bash`).
+You may also need to split some multi-command layers in the Dockerfile.
+If you see OOM errors during SuperBuild you should decrease CPU_RATIO (e.g.
+0.75).
+
+## Container examples
+
+```bash
+# Pull GPU image and create a new container with your home directory as volume
+# (requires apt package nvidia-docker2 and CUDA>=11.0)
+docker create --gpus=all --volume $HOME:/home/otbuser/volume -it \
+ --name otbtf-gpu mdl4eo/otbtf:3.3.2-gpu
+
+# Run interactive
+docker start -i otbtf-gpu
+
+# Run in background
+docker start otbtf-gpu
+docker exec otbtf-gpu \
+ python -c 'import tensorflow as tf; print(tf.test.is_gpu_available())'
+```
+
+### Rebuild OTB with more modules
+
+Enter a development ready docker image:
+
+```bash
+docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf:3.3.2-gpu-dev
+docker start -i otbtf-gpu-dev
+```
+
+Then, from the container shell:
+
+```bash
+sudo -i
+cd /src/otb/otb/Modules/Remote
+git clone https://gitlab.irstea.fr/raffaele.gaetano/otbSelectiveHaralickTextures.git
+cd /src/otb/build/OTB/build
+cmake -DModule_OTBAppSelectiveHaralickTextures=ON /src/otb/otb && make install -j
+```
+
+### Container with GUI
+
+GUI is disabled by default in order to save space, and because docker xvfb
+isn't working properly with OpenGL.
+OTB GUI seems OK but monteverdi isn't working
+
+```bash
+docker build --network='host' -t otbtf:cpu-gui \
+ --build-arg BASE_IMG=ubuntu:20.04 \
+ --build-arg GUI=true .
+docker create -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$DISPLAY -it \
+ --name otbtf-gui otbtf:cpu-gui
+docker start -i otbtf-gui
+$ mapla
+```
+
+## Common errors
+
+Build :
+
+```
+Error response from daemon:
+manifest for nvidia/cuda:11.0-cudnn8-devel-ubuntu20.04 not found:
+manifest unknown: manifest unknown
+```
+
+=> Image is missing from dockerhub
+
+Run :
+
+```
+failed call to cuInit:
+UNKNOWN ERROR (303) / no NVIDIA GPU device is present:
+/dev/nvidia0 does not exist
+```
+
+=> Nvidia driver is missing or disabled, make sure to add
+` --gpus=all` to your docker run or create command
diff --git a/doc/docker_troubleshooting.md b/doc/docker_troubleshooting.md
new file mode 100644
index 00000000..5835a6b9
--- /dev/null
+++ b/doc/docker_troubleshooting.md
@@ -0,0 +1,177 @@
+# Docker troubleshooting
+
+You can find plenty of help on the web about docker.
+This section only provides the basics for newcomers that are eager to use
+OTBTF!
+This section is largely inspired from the
+[moringa docker help](https://gitlab.irstea.fr/raffaele.gaetano/moringa/blob/develop/docker/README.md). Big thanks to them.
+
+## Useful diagnostic commands
+
+Here are some useful commands.
+
+```bash
+docker info # System info
+docker images # List local images
+docker container ls # List containers
+docker ps # Show running containers
+```
+
+On Linux, control state with `service`:
+
+```bash
+sudo service docker {status,enable,disable,start,stop,restart}
+```
+
+### Run some commands
+
+Run a simple command in a one-shot container:
+
+```bash
+docker run mdl4eo/otbtf:3.4.0-cpu otbcli_PatchesExtraction
+```
+
+You can also use the image in interactive mode with bash:
+
+```bash
+docker run -ti mdl4eo/otbtf:3.4.0-cpu bash
+```
+
+### Mounting file systems
+
+You can mount filesystem in the docker image.
+For instance, suppose you have some data in `/mnt/disk1/` that you want
+to use inside the container:
+
+The following command shows you how to access the folder from the docker image.
+
+```bash
+docker run -v /mnt/disk1/:/data/ -ti mdl4eo/otbtf:3.4.0-cpu bash -c "ls /data"
+```
+Beware of ownership issues! see the last section of this doc.
+
+### Persistent container
+
+Persistent (named) container with volume, here with home dir, but it can be
+any directory.
+
+```bash
+docker create --interactive --tty --volume /home/$USER:/home/otbuser/ \
+ --name otbtf mdl4eo/otbtf:3.4.0-cpu /bin/bash
+```
+
+!!! warning
+
+ Beware of ownership issues, see
+ [this section](#fix-volume-ownership-sissues).
+
+### Interactive session
+
+```bash
+docker start -i otbtf
+```
+
+### Background container
+
+```bash
+docker start otbtf
+docker exec otbtf ls -alh
+docker stop otbtf
+```
+
+### Running commands with root user
+
+Background container is one easy way:
+
+```bash
+docker start otbtf
+# Example with apt update
+# (you can't use &&, one docker exec is
+# required for each command)
+docker exec --user root otbtf apt-get update
+docker exec --user root otbtf apt-get upgrade -y
+```
+
+### Container-specific commands, especially for background containers:
+
+```bash
+docker inspect otbtf # See full container info dump
+docker logs otbtf # See command logs and outputs
+docker stats otbtf # Real time container statistics
+docker {pause,unpause} otbtf # Freeze container
+```
+
+### Stop a background container
+
+Don't forget to stop the container after you have done.
+
+```bash
+docker stop otbtf
+```
+
+### Remove a persistent container
+
+```bash
+docker rm otbtf
+```
+
+## Fix volume ownership issues
+
+Generally, this is required if host's UID > 1000.
+
+When mounting a volume, you may experience errors while trying to write files
+from within the container.
+Since the default user (**otbuser**) is UID 1000, you won't be able to write
+files into your volume
+which is mounted with the same UID than your linux host user (may be UID 1001
+or more).
+In order to address this, you need to edit the container's user UID and GID to
+match the right numerical value.
+This will only persist in a named container, it is required every time you're
+creating a new one.
+
+
+Create a named container (here with your HOME as volume), Docker will
+automatically pull image
+
+```bash
+docker create --interactive --tty --volume /home/$USER:/home/otbuser \
+ --name otbtf mdl4eo/otbtf:3.4.0-cpu /bin/bash
+```
+
+Start a background container process:
+
+```bash
+docker start otbtf
+```
+
+Exec required commands with user root (here with host's ID, replace $UID and
+$GID with desired values):
+
+```bash
+docker exec --user root otbtf usermod otbuser -u $UID
+docker exec --user root otbtf groupmod otbuser -g $GID
+```
+
+Force reset ownership with updated UID and GID.
+Make sure to double check that `docker exec otbtf id` because recursive chown
+will apply to your volume in `/home/otbuser`
+
+```bash
+docker exec --user root otbtf chown -R otbuser:otbuser /home/otbuser
+```
+
+Stop the background container and start a new interactive shell:
+
+```bash
+docker stop otbtf
+docker start -i otbtf
+```
+
+Check if ownership is right
+
+```bash
+id
+ls -Alh /home/otbuser
+touch /home/otbuser/test.txt
+```
diff --git a/doc/DOCKERUSE.md b/doc/docker_use.md
similarity index 66%
rename from doc/DOCKERUSE.md
rename to doc/docker_use.md
index 4c92b614..59990bef 100644
--- a/doc/DOCKERUSE.md
+++ b/doc/docker_use.md
@@ -1,9 +1,11 @@
-# OTBTF docker images overview
+# Install from docker
-### Available images
+## Latest images
-Here is the list of the latest OTBTF docker images hosted on [dockerhub](https://hub.docker.com/u/mdl4eo).
-Since OTBTF >= 3.2.1 you can find latest docker images on [gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry).
+Here is the list of the latest OTBTF docker images hosted on
+[dockerhub](https://hub.docker.com/u/mdl4eo).
+Since OTBTF >= 3.2.1 you can find latest docker images on
+[gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry).
| Name | Os | TF | OTB | Description | Dev files | Compute capability |
|------------------------------------------------------------------------------------| ------------- | ------ |-------| ---------------------- | --------- | ------------------ |
@@ -14,37 +16,25 @@ Since OTBTF >= 3.2.1 you can find latest docker images on [gitlab.irstea.fr](htt
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.4.0-gpu-opt** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.4.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-The list of older releases is available [here](#older-docker-releases).
+The list of older releases is available [here](#older-images).
-You can also find more interesting OTBTF flavored images at [LaTelescop gitlab registry](https://gitlab.com/latelescop/docker/otbtf/container_registry/).
+!!! warning
+ Until r2.4, all images are development-ready, and the sources are located
+ in `/work/`.
+ Since r2.4, development-ready images have the source in `/src/` and are
+ tagged "...-dev".
+## Build your own images
-### Development ready images
-
-Until r2.4, all images are development-ready, and the sources are located in `/work/`.
-Since r2.4, development-ready images have the source in `/src/`.
-
-### Build your own images
-
-If you want to use optimization flags, change GPUs compute capability, etc. you can build your own docker image using the provided dockerfile.
+If you want to use optimization flags, change GPUs compute capability, etc.
+you can build your own docker image using the provided dockerfile.
See the [docker build documentation](../tools/docker/README.md).
-# Mounting file systems
-
-You can mount filesystem in the docker image.
-For instance, suppose you have some data in `/mnt/my_device/` that you want to use inside the container:
-
-The following command shows you how to access the folder from the docker image.
-
-```bash
-docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf:3.4.0-cpu bash -c "ls /data"
-```
-Beware of ownership issues! see the last section of this doc.
-
-# GPU enabled docker
+## GPU enabled docker
In Linux, this is quite straightforward.
-Just follow the steps described in the [nvidia-docker documentation](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html).
+Just follow the steps described in the
+[nvidia-docker documentation](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html).
You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime** :
With Docker version earlier than 19.03 :
@@ -59,12 +49,22 @@ With Docker version including and after 19.03 :
docker run --gpus all -ti mdl4eo/otbtf:3.4.0-gpu bash
```
-You can find some details on the **GPU docker image** and some **docker tips and tricks** on [this blog](https://mdl4eo.irstea.fr/2019/10/15/otbtf-docker-image-with-gpu/).
+You can find some details on the **GPU docker image** and some **docker tips
+and tricks** on
+[this blog](https://mdl4eo.irstea.fr/2019/10/15/otbtf-docker-image-with-gpu/).
Be careful though, these infos might be a bit outdated...
-# Docker Installation
+## Docker Installation
+
+This section is a very small insight on the installation of docker on Linux
+and Windows.
+
+### Debian and Ubuntu
+
+See here how to install docker on Ubuntu
+[here](https://docs.docker.com/engine/install/ubuntu/).
-### Installation and first steps on Windows 10
+### Windows 10
1. Install [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10#manual-installation-steps) (Windows Subsystem for Linux)
2. Install [docker desktop](https://www.docker.com/products/docker-desktop)
@@ -79,163 +79,14 @@ Troubleshooting:
- [Docker for windows WSL documentation](https://docs.docker.com/docker-for-windows/wsl)
- [WSL2 installation steps](https://docs.microsoft.com/en-us/windows/wsl/install-win10)
-### Use the GPU with Windows 10 + WSL2
-
-*Work in progress*
-
-Some users have reported to use OTBTF with GPU in windows 10 using WSL2.
-How to install WSL2 with Cuda on windows 10:
-https://docs.nvidia.com/cuda/wsl-user-guide/index.html
-https://docs.docker.com/docker-for-windows/wsl/#gpu-support
-
-
-### Debian and Ubuntu
-
-See here how to install docker on Ubuntu [here](https://docs.docker.com/engine/install/ubuntu/).
-
-# Docker Usage
-
-This section is largely inspired from the [moringa docker help](https://gitlab.irstea.fr/raffaele.gaetano/moringa/blob/develop/docker/README.md). Big thanks to them.
-
-## Useful diagnostic commands
-
-Here are some useful commands.
-
-```bash
-docker info # System info
-docker images # List local images
-docker container ls # List containers
-docker ps # Show running containers
-```
-
-On Linux, control state with systemd:
-```bash
-sudo systemctl {status,enable,disable,start,stop} docker
-```
-
-### Run some commands
-
-Run a simple command in a one-shot container:
-
-```bash
-docker run mdl4eo/otbtf:3.4.0-cpu otbcli_PatchesExtraction
-```
-
-You can also use the image in interactive mode with bash:
-```bash
-docker run -ti mdl4eo/otbtf:3.4.0-cpu bash
-```
-
-### Persistent container
-
-Persistent (named) container with volume, here with home dir, but it can be any directory.
-Beware of ownership issues, see the last section of this doc.
-
-```bash
-docker create --interactive --tty --volume /home/$USER:/home/otbuser/ \
- --name otbtf mdl4eo/otbtf:3.4.0-cpu /bin/bash
-```
-
-### Interactive session
-
-```bash
-docker start -i otbtf
-```
-
-### Background container
+!!! Info
-```bash
-docker start otbtf
-docker exec otbtf ls -alh
-docker stop otbtf
-```
-
-### Running commands with root user
-
-Background container is the easiest way:
-
-```bash
-docker start otbtf
-# Example with apt update (you can't use &&, one docker exec is required for each command)
-docker exec --user root otbtf apt-get update
-docker exec --user root otbtf apt-get upgrade -y
-```
-
-### Container-specific commands, especially for background containers:
-
-```bash
-docker inspect otbtf # See full container info dump
-docker logs otbtf # See command logs and outputs
-docker stats otbtf # Real time container statistics
-docker {pause,unpause} otbtf # Freeze container
-```
-
-### Stop a background container
-
-Don't forget to stop the container after you have done.
-
-```bash
-docker stop otbtf
-```
-
-### Remove a persistent container
-
-```bash
-docker rm otbtf
-```
-
-# Fix volume ownership issue (required if host's UID > 1000)
-
-When mounting a volume, you may experience errors while trying to write files from within the container.
-Since the default user (**otbuser**) is UID 1000, you won't be able to write files into your volume
-which is mounted with the same UID than your linux host user (may be UID 1001 or more).
-In order to address this, you need to edit the container's user UID and GID to match the right numerical value.
-This will only persist in a named container, it is required every time you're creating a new one.
-
-
-Create a named container (here with your HOME as volume), Docker will automatically pull image
-
-```bash
-docker create --interactive --tty --volume /home/$USER:/home/otbuser \
- --name otbtf mdl4eo/otbtf:3.4.0-cpu /bin/bash
-```
-
-Start a background container process:
-
-```bash
-docker start otbtf
-```
-
-Exec required commands with user root (here with host's ID, replace $UID and $GID with desired values):
-
-```bash
-docker exec --user root otbtf usermod otbuser -u $UID
-docker exec --user root otbtf groupmod otbuser -g $GID
-```
-
-Force reset ownership with updated UID and GID.
-Make sure to double check that `docker exec otbtf id` because recursive chown will apply to your volume in `/home/otbuser`
-
-```bash
-docker exec --user root otbtf chown -R otbuser:otbuser /home/otbuser
-```
-
-Stop the background container and start a new interactive shell:
-
-```bash
-docker stop otbtf
-docker start -i otbtf
-```
-
-Check if ownership is right
-
-```bash
-id
-ls -Alh /home/otbuser
-touch /home/otbuser/test.txt
-```
+ Some users have reported to use OTBTF with GPU in windows 10 using WSL2.
+ How to install WSL2 with Cuda on windows 10:
+ https://docs.nvidia.com/cuda/wsl-user-guide/index.html
+ https://docs.docker.com/docker-for-windows/wsl/#gpu-support
-# Older docker releases
+## Older images
Here you can find the list of older releases of OTBTF:
diff --git a/doc/gen_ref_pages.py b/doc/gen_ref_pages.py
new file mode 100644
index 00000000..a02b8180
--- /dev/null
+++ b/doc/gen_ref_pages.py
@@ -0,0 +1,25 @@
+"""Generate the code reference pages."""
+
+from pathlib import Path
+
+import mkdocs_gen_files
+
+for path in sorted(Path("otbtf").rglob("*.py")): #
+ module_path = path.relative_to(".").with_suffix("") #
+ doc_path = path.relative_to(".").with_suffix(".md") #
+ full_doc_path = Path("reference", doc_path) #
+
+ parts = list(module_path.parts)
+
+ if parts[-1] == "__init__": #
+ parts = parts[:-1]
+ elif parts[-1] == "__main__":
+ continue
+
+ with mkdocs_gen_files.open(full_doc_path, "w") as fd: #
+ identifier = ".".join(parts) #
+ print("::: " + identifier)
+ print("::: " + identifier, file=fd) #
+
+ mkdocs_gen_files.set_edit_path(full_doc_path, path)
+
diff --git a/doc/images/modelbase.png b/doc/images/modelbase.png
new file mode 100644
index 00000000..1b5807d5
--- /dev/null
+++ b/doc/images/modelbase.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbb9e80568c04da7f96423ce7bebe25e6289e57bd9ccff8a6f183b4eddead2f0
+size 103824
diff --git a/doc/images/modelbase_1.png b/doc/images/modelbase_1.png
new file mode 100644
index 00000000..39fe2f30
--- /dev/null
+++ b/doc/images/modelbase_1.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd13daaf55b097fb1bb2c4834fad45c267436db7d308684543785aba689b3957
+size 94962
diff --git a/doc/images/modelbase_2.png b/doc/images/modelbase_2.png
new file mode 100644
index 00000000..14ff47b5
--- /dev/null
+++ b/doc/images/modelbase_2.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64c81033c7d14432fbac517ed7d18ff9b5fd45999b4dca0168d9b070cd8baee9
+size 95149
diff --git a/doc/images/modelbase_3.png b/doc/images/modelbase_3.png
new file mode 100644
index 00000000..4cf48fb4
--- /dev/null
+++ b/doc/images/modelbase_3.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d17b96e940ed7957fb08c02485c6e103124582bd815e857728365e9c9dcaca35
+size 88527
diff --git a/doc/images/modelbase_4.png b/doc/images/modelbase_4.png
new file mode 100644
index 00000000..1b442d9a
--- /dev/null
+++ b/doc/images/modelbase_4.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9647cd300df533b1550170da2cf500f17c914cb77d7b61c9585249501ae54d2e
+size 151715
diff --git a/doc/images/pipeline.png b/doc/images/pipeline.png
new file mode 100644
index 00000000..546ac7c5
--- /dev/null
+++ b/doc/images/pipeline.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:927369acc474e03d388c5efb3d142e4657ad8c28dbbd1195a6ba370f7dcd2d40
+size 153406
diff --git a/doc/index.md b/doc/index.md
new file mode 100644
index 00000000..3f30a209
--- /dev/null
+++ b/doc/index.md
@@ -0,0 +1,112 @@
+# OTBTF: Orfeo ToolBox meets TensorFlow
+
+<p align="center">
+<img src="images/logo.png" width="160px">
+<br>
+<a href="https://gitlab.irstea.fr/remi.cresson/otbtf/-/releases">
+<img src="https://gitlab.irstea.fr/remi.cresson/otbtf/-/badges/release.svg">
+</a>
+<a href="https://gitlab.irstea.fr/remi.cresson/otbtf/-/commits/master">
+<img src="https://gitlab.irstea.fr/remi.cresson/otbtf/badges/master/pipeline.svg">
+</a>
+<a href="LICENSE">
+<img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg">
+</a>
+</p>
+
+
+This remote module of the [Orfeo ToolBox](https://www.orfeo-toolbox.org)
+provides a generic, multi purpose deep learning framework, targeting remote
+sensing images processing. It contains a set of new process objects for OTB
+that internally invoke [Tensorflow](https://www.tensorflow.org/), and new [OTB
+applications](#otb-applications) to perform deep learning with real-world
+remote sensing images. Applications can be used to build OTB pipelines from
+Python or C++ APIs. OTBTF also includes a [python API](#python-api) to build
+Keras compliant models, easy to train in distributed environments.
+
+
+## Features
+
+### OTB Applications
+
+- Sample patches in remote sensing images with `PatchesExtraction`,
+- Inference with support of OTB streaming mechanism with
+`TensorflowModelServe`: this means that inference is not limited by images
+number, size, of channels depths, and can be used as a "lego" in any pipeline
+composed of OTB applications and preserving streaming.
+- Model training, supporting save/restore/import operations (a model can be
+trained from scratch or fine-tuned) with `TensorflowModelTrain`. This
+application targets mostly newcomers and is nice for educational purpose, but
+deep learning practitioners will for sure prefer the Python API of OTBTF.
+
+### Python API
+
+The `otbtf` module targets python developers that want to train their own
+model from python with TensorFlow or Keras.
+It provides various classes for datasets and iterators to handle the
+_patches images_ generated from the `PatchesExtraction` OTB application.
+For instance, the `otbtf.DatasetFromPatchesImages` can be instantiated from a
+set of _patches images_ and delivering samples as `tf.dataset` that can be
+used in your favorite TensorFlow pipelines, or convert your patches into
+TFRecords. The `otbtf.TFRecords` enables you train networks from TFRecords
+files, which is quite suited for distributed training. Read more in the
+[tutorial for keras](otbtf/examples/tensorflow_v2x/fcnn/README.md).
+
+## Examples
+
+Below are some screen captures of deep learning applications performed at
+large scale with OTBTF.
+ - Landcover mapping (Spot-7 images --> Building map using semantic
+segmentation)
+
+
+
+ - Super resolution (Sentinel-2 images upsampled with the
+[SR4RS software](https://github.com/remicres/sr4rs), which is based on OTBTF)
+
+
+
+ - Sentinel-2 reconstruction with Sentinel-1 VV/VH with the
+[Decloud software](https://github.com/CNES/decloud), which is based on OTBTF
+
+
+
+ - Image to image translation (Spot-7 image --> Wikimedia Map using CGAN.
+So unnecessary but fun!)
+
+
+
+## Contribute
+
+Every one can **contribute** to OTBTF. Just open a PR :)
+
+## Cite
+
+```
+@article{cresson2018framework,
+ title={A framework for remote sensing images processing using deep learning techniques},
+ author={Cresson, R{\'e}mi},
+ journal={IEEE Geoscience and Remote Sensing Letters},
+ volume={16},
+ number={1},
+ pages={25--29},
+ year={2018},
+ publisher={IEEE}
+}
+```
+
+## Additional resources
+
+- A small [tutorial](https://mdl4eo.irstea.fr/2019/01/04/an-introduction-to-deep-learning-on-remote-sensing-images-tutorial/) on MDL4EO's blog
+- in the `python` folder are provided some [ready-to-use deep networks, with
+documentation and scientific references](doc/EXAMPLES.md).
+- A [book](https://doi.org/10.1201/9781003020851): *Cresson, R. (2020).
+Deep Learning for Remote Sensing Images with Open Source Software. CRC Press.*
+Use QGIS, OTB and Tensorflow to perform various kind of deep learning sorcery
+on remote sensing images (patch-based classification for landcover mapping,
+semantic segmentation of buildings, optical image restoration from joint
+SAR/Optical time series).
+- Check [our repository](https://github.com/remicres/otbtf_tutorials_resources)
+containing stuff (data and models) to begin with with!
+- Finally, take a look in the `test` folder. You will find plenty of command
+lines for applications tests!
diff --git a/mkdocs.yml b/mkdocs.yml
new file mode 100644
index 00000000..c1603ff8
--- /dev/null
+++ b/mkdocs.yml
@@ -0,0 +1,80 @@
+# mkdocs.yml
+theme:
+ name: "material"
+ logo: images/logo.png
+ icon:
+ repo: fontawesome/brands/github
+# admonition:
+# <type>: <icon>
+ features:
+ - navigation.instant
+ - content.code.copy
+ - content.code.annotate
+ - toc.follow
+# analytics:
+# - gtag: G-PD85X2X108
+# custom_dir: doc/custom_theme
+
+plugins:
+- search
+- gen-files:
+ scripts:
+ - doc/gen_ref_pages.py
+- mkdocstrings:
+ watch:
+ - otbtf/
+- literate-nav:
+ nav_file: SUMMARY.md
+- section-index
+- mermaid2
+
+nav:
+- Home: index.md
+- Installation:
+ - Install from docker: docker_use.md
+ - Build your own docker images: docker_build.md
+ - Docker troubleshooting: docker_troubleshooting.md
+ - Build from source: build_from_sources.md
+- Applications:
+ - Overview: app_overview.md
+ - Sampling: app_sampling.md
+ - Training: app_training.md
+ - Inference: app_inference.md
+- Python API:
+ - Model generalities: api_model_generalities.md
+ - Deterministic models: reference/otbtf/examples/tensorflow_v2x/deterministic/__init__.md
+ - Build and train deep learning models: api_tutorial.md
+- Tensorflow API v1 (deprecated):
+ - A few deep learning models: reference/otbtf/examples/tensorflow_v1x/__init__.md
+- Python API references:
+ - dataset: reference/otbtf/dataset.md
+ - tfrecords: reference/otbtf/tfrecords.md
+ - model: reference/otbtf/model.md
+
+# Customization
+extra:
+ feature:
+ tabs: true
+ social:
+ - icon: fontawesome/brands/github
+ link: https://github.com/remicres/otbtf
+use_directory_urls: false # this creates some otbtf/core.html pages instead of otbtf/core/index.html
+
+markdown_extensions:
+ - admonition
+ - toc:
+ permalink: true
+ title: On this page
+ toc_depth: 1-2
+ - pymdownx.highlight:
+ anchor_linenums: true
+ - pymdownx.inlinehilite
+ - pymdownx.snippets
+ - pymdownx.details
+ - pymdownx.superfences
+
+# Rest of the navigation..
+site_name: "OTBTF"
+repo_url: https://github.com/remicres/otbtf
+repo_name: otbtf
+docs_dir: doc/
diff --git a/otbtf/examples/__init__.py b/otbtf/examples/__init__.py
new file mode 100644
index 00000000..c49c8f05
--- /dev/null
+++ b/otbtf/examples/__init__.py
@@ -0,0 +1,4 @@
+"""
+# Examples
+
+"""
diff --git a/doc/EXAMPLES.md b/otbtf/examples/tensorflow_v1x/__init__.py
similarity index 50%
rename from doc/EXAMPLES.md
rename to otbtf/examples/tensorflow_v1x/__init__.py
index d48c56a9..2ed0ba3a 100644
--- a/doc/EXAMPLES.md
+++ b/otbtf/examples/tensorflow_v1x/__init__.py
@@ -1,16 +1,31 @@
-# Examples
+"""
-Some examples of ready-to-use deep learning architectures built with the TensorFlow API from python.
-All models used are provided in this directory.
+This section provides some examples of ready-to-use deep learning
+architectures built with the TensorFlow API from python. All models used are
+provided in the `otbtf/examples/tensorflow_v1x/` directory.
+
+!!! note
+
+ While we recommend to use the Tensorflow API v2, the following examples are
+ still great to understand how the models work, particulary in inference
+ mode. Models based on the TF API v1 can be trained easily with the
+ `TensorflowModelTrain` application, which allows beginners to focus more
+ on the overall process rather than the code.
+
+!!! warning
+
+ `tricks.py` is here for backward compatibility with codes based on OTBTF 1.x
+ and 2.x.
**Table of Contents**
-1. [Simple CNN](#part1)
-2. [Fully convolutional network](#part2)
-3. [M3Fusion Model](#part3)
-4. [Maggiori model](#part4)
-5. [Fully convolutional network with separate Pan/MS channels](#part5)
-## Simple CNN <a name="part1"></a>
+1. [Simple CNN](#simple-cnn)
+2. [Fully convolutional network](#fully-convolutional-network)
+3. [M3Fusion Model](#m3fusion-model)
+4. [Maggiori model](#maggiori-model)
+5. [Fully convolutional network with separate channels](#fully-convolutional-network-with-separate-channels)
+
+# Simple CNN
This simple model estimates the class of an input patch of image.
This model consists in successive convolutions/pooling/relu of the input (*x* placeholder).
@@ -23,7 +38,7 @@ The following figure summarizes this architecture.
<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_cnn.png" />
-### Generate the model
+## Generate the model
Use the python script to generate the SavedModel that will be used by OTBTF applications.
@@ -33,16 +48,18 @@ python create_savedmodel_simple_cnn.py --outdir $modeldir
Note that you can adjust the number of classes for the model with the `--nclasses` option.
-### Train the model
+## Train the model with the CLI
-Use **TensorflowModelTrain** to train this model.
+Use **TensorflowModelTrain** in the command line interface to train this model.
-```
+```commandLine
otbcli_TensorflowModelTrain \
-model.dir $modeldir \
-model.saveto "$modeldir/variables/variables" \
--training.source1.il $patches_train -training.source1.patchsizex 1 -training.source1.patchsizey 1 -training.source1.placeholder "x" \
--training.source2.il $labels_train -training.source2.patchsizex 1 -training.source2.patchsizey 1 -training.source2.placeholder "y" \
+-training.source1.il $patches_train -training.source1.placeholder "x" \
+-training.source1.patchsizex 1 -training.source1.patchsizey 1 \
+-training.source2.il $labels_train -training.source2.placeholder "y" \
+-training.source2.patchsizex 1 -training.source2.patchsizey 1 \
-training.targetnodes "optimizer" \
-validation.mode "class" \
-validation.source1.il $patches_valid -validation.source1.name "x" \
@@ -51,19 +68,25 @@ otbcli_TensorflowModelTrain \
Type `otbcli_TensorflowModelTrain --help` to display the help.
-For instance, you can change the number of epochs to 50 with `-training.epochs 50` or you can change the batch size to 8 with `-training.batchsize 8`.
-In addition, it is possible to feed some scalar values to scalar placeholder of the model (currently, bool, int and float are supported).
-For instance, our model has a placeholder called *lr* that controls the learning rate of the optimizer.
-We can change this value at runtime using `-training.userplaceholders "lr=0.0002"`
+You can change the number of epochs to 50 with `-training.epochs 50` or you
+can change the batch size to 8 with `-training.batchsize 8`.
+In addition, it is possible to feed some scalar values to scalar placeholder
+of the model (currently, bool, int and float are supported).
+For instance, our model has a placeholder called *lr* that controls the
+learning rate of the optimizer.
+We can change this value at runtime using
+`-training.userplaceholders "lr=0.0002"`
-### Inference
+## Inference
-This model can be used either in patch-based mode or in fully convolutional mode.
+This model can be used either in patch-based mode or in fully convolutional
+mode.
-#### Patch-based mode
+### Patch-based mode
You can estimate the class of every pixels of your input image.
-Since the model is able to estimate the class of the center value of a 16x16 patch, you can run the model over the whole image in patch-based mode.
+Since the model is able to estimate the class of the center value of a 16x16
+patch, you can run the model over the whole image in patch-based mode.
```
otbcli_TensorflowModelServe \
@@ -76,17 +99,22 @@ otbcli_TensorflowModelServe \
-out $output_classif
```
-However, patch-based approach is slow because each patch is processed independently, which is not computationally efficient.
+However, patch-based approach is slow because each patch is processed
+independently, which is not computationally efficient.
-#### Fully convolutional mode
+### Fully convolutional mode
-In fully convolutional mode, the model is used to process larger blocks in order to estimate simultaneously multiple pixels classes.
+In fully convolutional mode, the model is used to process larger blocks in
+order to estimate simultaneously multiple pixels classes.
The model has a total number of 4 strides (caused by pooling).
-Hence the physical spacing of the features maps, in spatial dimensions, is divided by 4.
-This is what is called *spcscale* in the **TensorflowModelServe** application.
-If you want to use the model in fully convolutional mode, you have to tell **TensorflowModelServe** that the model performs a change of physical spacing of the output, 4 in our case.
-
-```
+Hence the physical spacing of the features maps, in spatial dimensions, is
+divided by 4. This is what is called *spcscale* in the
+**TensorflowModelServe** application.
+If you want to use the model in fully convolutional mode, you have to tell
+**TensorflowModelServe** that the model performs a change of physical spacing
+of the output, 4 in our case.
+
+```commandLine
otbcli_TensorflowModelServe \
-source1.il $image" \
-source1.rfieldx 16 \
@@ -99,18 +127,30 @@ otbcli_TensorflowModelServe \
-out $output_classif_fcn
```
-## Fully convolutional network <a name="part2"></a>
+# Fully convolutional network
-The `create_savedmodel_simple_fcn.py` script enables you to create a fully convolutional model which does not use any stride.
+The `create_savedmodel_simple_fcn.py` script enables you to create a fully
+convolutional model which does not use any stride.
<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_fcnn.png" />
-Thank to that, once trained this model can be applied on the image to produce a landcover map at the same resolution as the input image, in a fully convolutional (i.e. fast) manner.
-The main difference with the model described in the previous section is the *spcscale* parameter that must be let to default (i.e. unitary).
+Thanks to that, once trained this model can be applied on the image to produce
+a landcover map at the same resolution as the input image, in a fully
+convolutional (i.e. fast) manner.
+The main difference with the model described in the previous section is
+the *spcscale* parameter that must be let to default (i.e. unitary).
-Create the SavedModel using `python create_savedmodel_simple_fcn.py --outdir $modeldir` then train it as before.
-Then you can produce the land cover map at pixel level in fully convolutional mode:
+Create the SavedModel:
+
+```commandLine
+python create_savedmodel_simple_fcn.py --outdir $modeldir
```
+
+Then, train it as we saw before.
+Then you can produce the land cover map at pixel level in fully convolutional
+mode:
+
+```commandLine
otbcli_TensorflowModelServe \
-source1.il $image" \
-source1.rfieldx 16 \
@@ -122,33 +162,43 @@ otbcli_TensorflowModelServe \
-out $output_classif
```
-## M3Fusion Model <a name="part3"></a>
+# M3Fusion Model
-The M3Fusion model (stands for MultiScale/Multimodal/Multitemporal satellite data fusion) is a model designed to input time series and very high resolution images.
+The M3Fusion model (stands for MultiScale/Multimodal/Multitemporal satellite
+data fusion) is a model designed to input time series and very high resolution
+images.
-Benedetti, P., Ienco, D., Gaetano, R., Ose, K., Pensa, R. G., & Dupuy, S. (2018). _M3Fusion: A Deep Learning Architecture for Multiscale Multimodal Multitemporal Satellite Data Fusion_. IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing, 11(12), 4939-4949.
+Benedetti, P., Ienco, D., Gaetano, R., Ose, K., Pensa, R. G., & Dupuy, S.
+(2018). _M3Fusion: A Deep Learning Architecture for Multiscale Multimodal
+Multitemporal Satellite Data Fusion_. IEEE Journal of Selected Topics in
+Applied Earth Observations and Remote Sensing, 11(12), 4939-4949.
See the original paper [here](https://arxiv.org/pdf/1803.01945).
-The M3 model is patch-based, and process two input sources simultaneously: (i) time series, and (ii) a very high resolution image.
+The M3 model is patch-based, and process two input sources simultaneously:
+(i) time series, and (ii) a very high resolution image.
The output class estimation is performed at pixel level.
-### Generate the model
+## Generate the model
```
python create_savedmodel_ienco-m3_patchbased.py --outdir $modeldir
```
-Note that you can adjust the number of classes for the model with the `--nclasses` option.
-Type `python create_savedmodel_ienco-m3_patchbased.py --help` to see the other available parameters.
+Note that you can adjust the number of classes for the model with the
+`--nclasses` parameter.
+Type `python create_savedmodel_ienco-m3_patchbased.py --help` to see the other
+available parameters.
-### Train the model
+## Train the model from the CLI
-Let's train the M3 model from time series (TS) and Very High Resolution Satellite (VHRS) patches images.
+Let's train the M3 model from time series (TS) and Very High Resolution
+Satellite (VHRS) patches images.
<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/model_training.png" />
-First, tell OTBTF that we want two sources: one for time series + one for VHR image
+First, tell OTBTF that we want two sources: one for time series + one for
+VHR image
```
export OTB_TF_NSOURCES=2
@@ -156,131 +206,191 @@ export OTB_TF_NSOURCES=2
Run the **TensorflowModelTrain** application of OTBTF.
-Note that for time series we could also have provided a list of images rather that a single big images stack (since "sourceX.il" is an input image list parameter).
+Note that for time series we could also have provided a list of images rather
+that a single big images stack (since "sourceX.il" is an input image list
+parameter).
```
otbcli_TensorflowModelTrain \
-model.dir $modeldir \
-model.saveto "$modeldir/variables/variables" \
--training.source1.il $patches_ts_train -training.source1.patchsizex 1 -training.source1.patchsizey 1 -training.source1.placeholder "x_rnn" \
--training.source2.il $patches_vhr_train -training.source2.patchsizex 25 -training.source2.patchsizey 25 -training.source2.placeholder "x_cnn" \
--training.source3.il $labels_train -training.source3.patchsizex 1 -training.source3.patchsizey 1 -training.source3.placeholder "y" \
+-training.source1.il $patches_ts_train \
+-training.source1.patchsizex 1 \
+-training.source1.patchsizey 1 \
+-training.source1.placeholder "x_rnn" \
+-training.source2.il $patches_vhr_train \
+-training.source2.patchsizex 25 \
+-training.source2.patchsizey 25 \
+-training.source2.placeholder "x_cnn" \
+-training.source3.il $labels_train \
+-training.source3.patchsizex 1 \
+-training.source3.patchsizey 1 \
+-training.source3.placeholder "y" \
-training.targetnodes "optimizer" \
-training.userplaceholders "is_training=true" "drop_rate=0.1" "learning_rate=0.0002" \
-validation.mode "class" -validation.step 1 \
--validation.source1.il $patches_ts_valid -validation.source1.name "x_rnn" \
--validation.source2.il $patches_vhr_valid -validation.source2.name "x_cnn" \
--validation.source3.il $labels_valid -validation.source3.name "prediction"
+-validation.source1.il $patches_ts_valid \
+-validation.source1.name "x_rnn" \
+-validation.source2.il $patches_vhr_valid \
+-validation.source2.name "x_cnn" \
+-validation.source3.il $labels_valid \
+-validation.source3.name "prediction"
```
-### Inference
+## Inference
-Let's produce a land cover map using the M3 model from time series (TS) and Very High Resolution Satellite image (VHRS)
+Let's produce a land cover map using the M3 model from time series (TS) and
+Very High Resolution Satellite image (VHRS)
<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/classif_map.png" />
-Since we provide time series as the reference source (*source1*), the output classes are estimated at the same resolution.
-This model can be run in patch-based mode only.
+Since we provide time series as the reference source (*source1*), the output
+classes are estimated at the same resolution. This model can be run in
+patch-based mode only.
-```
+```commandLine
otbcli_TensorflowModelServe \
--source1.il $ts -source1.rfieldx 1 -source1.rfieldy 1 -source1.placeholder "x_rnn" \
--source2.il $vhr -source2.rfieldx 25 -source2.rfieldy 25 -source2.placeholder "x_cnn" \
+-source1.il $ts \
+-source1.rfieldx 1 -source1.rfieldy 1 \
+-source1.placeholder "x_rnn" \
+-source2.il $vhr \
+-source2.rfieldx 25 -source2.rfieldy 25 \
+-source2.placeholder "x_cnn" \
-model.dir $modeldir \
-output.names "prediction" -out $output_classif
```
-## Maggiori model <a name="part4"></a>
+# Maggiori model
-This architecture was one of the first to introduce a fully convolutional model suited for large scale remote sensing images.
+This architecture was one of the first to introduce a fully convolutional
+model suited for large scale remote sensing images.
-Maggiori, E., Tarabalka, Y., Charpiat, G., & Alliez, P. (2016). _Convolutional neural networks for large-scale remote-sensing image classification_. IEEE Transactions on Geoscience and Remote Sensing, 55(2), 645-657.
+Maggiori, E., Tarabalka, Y., Charpiat, G., & Alliez, P. (2016).
+_Convolutional neural networks for large-scale remote-sensing image
+classification_. IEEE Transactions on Geoscience and Remote Sensing, 55(2),
+645-657.
See the original paper [here](https://hal.inria.fr/hal-01350706/document).
-This fully convolutional model performs binary semantic segmentation of large scale images without any blocking artifacts.
+This fully convolutional model performs binary semantic segmentation of large
+scale images without any blocking artifacts.
-### Generate the model
+## Generate the model
-```
+```commandLine
python create_savedmodel_maggiori17_fullyconv.py --outdir $modeldir
```
-You can change the number of spectral bands of the input image that is processed with the model, using the `--n_channels` option.
+You can change the number of spectral bands of the input image that is
+processed with the model, using the `--n_channels` option.
-### Train the model
+## Train the model
The model perform the semantic segmentation from one single source.
-```
+```commandLine
otbcli_TensorflowModelTrain \
-model.dir $modeldir \
-model.saveto "$modeldir/variables/variables" \
--training.source1.il $patches_image_train -training.source1.patchsizex 80 -training.source1.patchsizey 80 -training.source1.placeholder "x" \
--training.source2.il $patches_labels_train -training.source2.patchsizex 16 -training.source2.patchsizey 16 -training.source2.placeholder "y" \
+-training.source1.il $patches_image_train \
+-training.source1.patchsizex 80 -training.source1.patchsizey 80 \
+-training.source1.placeholder "x" \
+-training.source2.il $patches_labels_train \
+-training.source2.patchsizex 16 -training.source2.patchsizey 16 \
+-training.source2.placeholder "y" \
-training.targetnodes "optimizer" \
-training.userplaceholders "is_training=true" "learning_rate=0.0002" \
-validation.mode "class" -validation.step 1 \
--validation.source1.il $patches_image_valid -validation.source1.name "x" \
--validation.source2.il $patches_labels_valid -validation.source2.name "estimated" \
+-validation.source1.il $patches_image_valid \
+-validation.source1.name "x" \
+-validation.source2.il $patches_labels_valid \
+-validation.source2.name "estimated"
```
-Note that the `userplaceholders` parameter contains the *is_training* placeholder, fed with value *true* because the default value for this placeholder is *false*, and it is used in the batch normalization layers (take a look in the `create_savedmodel_maggiori17_fullyconv.py` code).
+Note that the `userplaceholders` parameter contains the *is_training*
+placeholder, fed with value *true* because the default value for this
+placeholder is *false*, and it is used in the batch normalization layers (take
+a look in the `create_savedmodel_maggiori17_fullyconv.py` code).
-### Inference
+## Inference
This model can be used in fully convolutional mode only.
-This model performs convolutions with stride (i.e. downsampling), followed with transposed convolutions with strides (i.e. upsampling).
-Since there is no change of physical spacing (because downsampling and upsampling have both the same number of strides), the *spcscale* parameter is let to default (i.e. unitary).
-The receptive field of the model is 80x80, and the expression field is 16x16, due to the fact that the model keeps only the exact part of the output features maps.
+This model performs convolutions with stride (i.e. downsampling), followed
+with transposed convolutions with strides (i.e. upsampling).
+Since there is no change of physical spacing (because downsampling and
+upsampling have both the same number of strides), the *spcscale* parameter is
+let to default (i.e. unitary).
+The receptive field of the model is 80x80, and the expression field is 16x16,
+due to the fact that the model keeps only the exact part of the output
+features maps.
```
otbcli_TensorflowModelServe \
--source1.il $image -source1.rfieldx 80 -source1.rfieldy 80 -source1.placeholder x \
+-source1.il $image \
+-source1.rfieldx 80 -source1.rfieldy 80 \
+-source1.placeholder x \
-model.dir $modeldir \
-model.fullyconv on \
--output.names "estimated" -output.efieldx 16 -output.efieldy 16 \
+-output.names "estimated" \
+-output.efieldx 16 -output.efieldy 16 \
-out $output_classif
```
-## Fully convolutional network with separate Pan/MS channels <a name="part5"></a>
+# Fully convolutional network with separate channels
-It's common that very high resolution products are composed with a panchromatic channel at high-resolution (Pan), and a multispectral image generally at lower resolution (MS).
+It's common that very high resolution products are composed with a
+panchromatic channel at high-resolution (Pan), and a multispectral image
+generally at lower resolution (MS).
This model inputs separately the two sources (Pan and MS) separately.
-See: Gaetano, R., Ienco, D., Ose, K., & Cresson, R. (2018). A two-branch CNN architecture for land cover classification of PAN and MS imagery. Remote Sensing, 10(11), 1746.
+See: Gaetano, R., Ienco, D., Ose, K., & Cresson, R. (2018). A two-branch CNN
+architecture for land cover classification of PAN and MS imagery. Remote
+Sensing, 10(11), 1746.
<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_pxs_fcn.png" />
Use `create_savedmodel_pxs_fcn.py` to generate this model.
-During training, the *x1* and *x2* placeholders must be fed respectively with patches of size 8x8 and 32x32.
-You can use this model in a fully convolutional way with receptive field of size 32 (for the Pan image) and 8 (for the MS image) and an unitary expression field (i.e. equal to 1).
-Don't forget to tell OTBTF that we want two sources: one for Ms image + one for Pan image
+During training, the *x1* and *x2* placeholders must be fed respectively with
+patches of size 8x8 and 32x32.
+You can use this model in a fully convolutional way with receptive field of
+size 32 (for the Pan image) and 8 (for the MS image) and an unitary expression
+field (i.e. equal to 1).
+Don't forget to tell OTBTF that we want two sources: one for Ms image + one
+for Pan image
```
export OTB_TF_NSOURCES=2
```
-### Inference at MS image resolution
+## Inference at MS image resolution
Here we perform the land cover map at the same resolution as the MS image.
Do do this, we set the MS image as the first source in the **TensorflowModelServe** application.
```
otbcli_TensorflowModelServe \
--source1.il $ms -source1.rfieldx 8 -source1.rfieldy 8 -source1.placeholder "x1" \
--source2.il $pan -source2.rfieldx 32 -source2.rfieldy 32 -source2.placeholder "x2" \
+-source1.il $ms \
+-source1.rfieldx 8 -source1.rfieldy 8 \
+-source1.placeholder "x1" \
+-source2.il $pan \
+-source2.rfieldx 32 -source2.rfieldy 32 \
+-source2.placeholder "x2" \
-model.dir $modeldir \
-model.fullyconv on \
-output.names "prediction" \
-out $output_classif
```
-Note that we could also have set the Pan image as the first source, and tell the application to use a *spcscale* of 4.
+Note that we could also have set the Pan image as the first source, and tell
+the application to use a *spcscale* of 4.
```
otbcli_TensorflowModelServe \
--source1.il $pan -source1.rfieldx 32 -source1.rfieldy 32 -source1.placeholder "x2" \
--source2.il $ms -source2.rfieldx 8 -source2.rfieldy 8 -source2.placeholder "x1" \
+-source1.il $pan \
+-source1.rfieldx 32 -source1.rfieldy 32 \
+-source1.placeholder "x2" \
+-source2.il $ms \
+-source2.rfieldx 8 -source2.rfieldy 8 \
+-source2.placeholder "x1" \
-model.dir $modeldir \
-model.fullyconv on \
-output.names "prediction" \
@@ -288,30 +398,42 @@ otbcli_TensorflowModelServe \
-out $output_classif
```
-### Inference at Pan image resolution
+## Inference at Pan image resolution
Here we perform the land cover map at the same resolution as the Pan image.
-Do do this, we set the Pan image as the first source in the **TensorflowModelServe** application.
-Note that this model can not be applied in a fully convolutional fashion at the Pan image resolution.
+Do do this, we set the Pan image as the first source in the
+**TensorflowModelServe** application.
+Note that this model can not be applied in a fully convolutional fashion at
+the Pan image resolution.
We hence perform the processing in patch-based mode.
```
otbcli_TensorflowModelServe \
--source1.il $pan -source1.rfieldx 32 -source1.rfieldy 32 -source1.placeholder "x2" \
--source2.il $ms -source2.rfieldx 8 -source2.rfieldy 8 -source2.placeholder "x1" \
+-source1.il $pan \
+-source1.rfieldx 32 -source1.rfieldy 32 \
+-source1.placeholder "x2" \
+-source2.il $ms \
+-source2.rfieldx 8 -source2.rfieldy 8 \
+-source2.placeholder "x1" \
-model.dir $modeldir \
-output.names "prediction" \
-out $output_classif
```
-Note that we could also have set the MS image as the first source, and tell the application to use a *spcscale* of 0.25.
+Note that we could also have set the MS image as the first source, and tell
+the application to use a *spcscale* of 0.25.
```
otbcli_TensorflowModelServe \
--source1.il $ms -source1.rfieldx 8 -source1.rfieldy 8 -source1.placeholder "x1" \
--source2.il $pan -source2.rfieldx 32 -source2.rfieldy 32 -source2.placeholder "x2" \
+-source1.il $ms \
+-source1.rfieldx 8 -source1.rfieldy 8 \
+-source1.placeholder "x1" \
+-source2.il $pan \
+-source2.rfieldx 32 -source2.rfieldy 32 \
+-source2.placeholder "x2" \
-model.dir $modeldir \
-model.fullyconv on \
-output.names "prediction" \
-out $output_classif
```
+"""
diff --git a/otbtf/examples/tensorflow_v1x/create_savedmodel_ienco-m3_patchbased.py b/otbtf/examples/tensorflow_v1x/create_savedmodel_ienco-m3_patchbased.py
index 2a3ad56f..1edfe44d 100755
--- a/otbtf/examples/tensorflow_v1x/create_savedmodel_ienco-m3_patchbased.py
+++ b/otbtf/examples/tensorflow_v1x/create_savedmodel_ienco-m3_patchbased.py
@@ -21,29 +21,19 @@
# Reference:
#
-# Benedetti, P., Ienco, D., Gaetano, R., Ose, K., Pensa, R. G., & Dupuy, S. (2018)
-# M3Fusion: A Deep Learning Architecture for Multiscale Multimodal Multitemporal
-# Satellite Data Fusion. IEEE Journal of Selected Topics in Applied Earth
-# Observations and Remote Sensing, 11(12), 4939-4949.
+# Benedetti, P., Ienco, D., Gaetano, R., Ose, K., Pensa, R. G., & Dupuy, S.
+# (2018) M3Fusion: A Deep Learning Architecture for Multiscale Multimodal
+# Multitemporal Satellite Data Fusion. IEEE Journal of Selected Topics in
+# Applied Earth Observations and Remote Sensing, 11(12), 4939-4949.
import argparse
-from tricks import create_savedmodel
+
import tensorflow.compat.v1 as tf
import tensorflow.compat.v1.nn.rnn_cell as rnn
-tf.disable_v2_behavior()
+from tricks import create_savedmodel
-parser = argparse.ArgumentParser()
-parser.add_argument("--nunits", type=int, default=1024, help="number of units")
-parser.add_argument("--n_levels_lstm", type=int, default=1, help="number of lstm levels")
-parser.add_argument("--hm_epochs", type=int, default=400, help="hm epochs")
-parser.add_argument("--n_timestamps", type=int, default=37, help="number of images in timeseries")
-parser.add_argument("--n_dims", type=int, default=16, help="number of channels in timeseries images")
-parser.add_argument("--patch_window", type=int, default=25, help="patch size for the high-res image")
-parser.add_argument("--n_channels", type=int, default=4, help="number of channels in the high-res image")
-parser.add_argument("--nclasses", type=int, default=8, help="number of classes")
-parser.add_argument("--outdir", help="Output directory for SavedModel", required=True)
-params = parser.parse_args()
+tf.disable_v2_behavior()
def RnnAttention(x, nunits, nlayer, n_dims, n_timetamps, is_training_ph):
@@ -53,10 +43,12 @@ def RnnAttention(x, nunits, nlayer, n_dims, n_timetamps, is_training_ph):
# (before unstack) x is 1 tensor of shape [N, n_dims, n_timestamps]
x = tf.unstack(x, n_timetamps, axis=2)
- # (after unstack) x is a list of "n_timestamps" tensors of shape: [N, n_dims]
+ # (after unstack) x is a list of "n_timestamps" tensors of shape:
+ # [N, n_dims]
# NETWORK DEF
- # MORE THEN ONE LAYER: list of LSTMcell,nunits hidden units each, for each layer
+ # MORE THEN ONE LAYER: list of LSTMcell,nunits hidden units each, for
+ # each layer
if nlayer > 1:
cells = []
for _ in range(nlayer):
@@ -73,14 +65,17 @@ def RnnAttention(x, nunits, nlayer, n_dims, n_timetamps, is_training_ph):
# Trainable parameters
attention_size = nunits # int(nunits / 2)
- W_omega = tf.Variable(tf.random_normal([nunits, attention_size], stddev=0.1))
+ W_omega = tf.Variable(
+ tf.random_normal([nunits, attention_size], stddev=0.1))
b_omega = tf.Variable(tf.random_normal([attention_size], stddev=0.1))
u_omega = tf.Variable(tf.random_normal([attention_size], stddev=0.1))
- # Applying fully connected layer with non-linear activation to each of the B*T timestamps;
+ # Applying fully connected layer with non-linear activation to each of
+ # the B*T timestamps;
# the shape of `v` is (B,T,D)*(D,A)=(B,T,A), where A=attention_size
v = tf.tanh(tf.tensordot(outputs, W_omega, axes=1) + b_omega)
- # For each of the timestamps its vector of size A from `v` is reduced with `u` vector
+ # For each of the timestamps its vector of size A from `v` is reduced
+ # with `u` vector
vu = tf.tensordot(v, u_omega, axes=1) # (B,T) shape
alphas = tf.nn.softmax(vu) # (B,T) shape also
@@ -102,14 +97,17 @@ def CNN(x, nunits):
conv1 = tf.compat.v1.layers.batch_normalization(conv1)
- pool1 = tf.compat.v1.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)
+ pool1 = tf.compat.v1.layers.max_pooling2d(
+ inputs=conv1, pool_size=[2, 2], strides=2
+ )
conv2 = tf.compat.v1.layers.conv2d(
inputs=pool1,
filters=nunits,
kernel_size=[3, 3],
padding="valid",
- activation=tf.nn.relu)
+ activation=tf.nn.relu
+ )
conv2 = tf.compat.v1.layers.batch_normalization(conv2)
@@ -118,7 +116,8 @@ def CNN(x, nunits):
filters=nunits,
kernel_size=[3, 3],
padding="same",
- activation=tf.nn.relu)
+ activation=tf.nn.relu
+ )
conv3 = tf.compat.v1.layers.batch_normalization(conv3)
@@ -129,7 +128,8 @@ def CNN(x, nunits):
filters=nunits,
kernel_size=[1, 1],
padding="valid",
- activation=tf.nn.relu)
+ activation=tf.nn.relu
+ )
conv4 = tf.compat.v1.layers.batch_normalization(conv4)
@@ -140,8 +140,18 @@ def CNN(x, nunits):
return cnn, tensor_shape[1].value
-def get_prediction(x_rnn, x_cnn, nunits, nlayer, nclasses, n_dims, n_timetamps):
- vec_rnn = RnnAttention(x_rnn, nunits, nlayer, n_dims, n_timetamps, is_training_ph)
+def get_prediction(
+ x_rnn,
+ x_cnn,
+ nunits,
+ nlayer,
+ nclasses,
+ n_dims,
+ n_timetamps
+):
+ vec_rnn = RnnAttention(
+ x_rnn, nunits, nlayer, n_dims, n_timetamps, is_training_ph
+ )
vec_cnn, cnn_dim = CNN(x_cnn, 512)
features_learnt = tf.concat([vec_rnn, vec_cnn], axis=1, name="features")
@@ -165,48 +175,144 @@ def get_prediction(x_rnn, x_cnn, nunits, nlayer, nclasses, n_dims, n_timetamps):
return pred_c1, pred_c2, pred_full, features_learnt
-# Create the TensorFlow graph
-with tf.compat.v1.Graph().as_default():
- x_rnn = tf.compat.v1.placeholder(tf.float32, [None, 1, 1, params.n_dims * params.n_timestamps], name="x_rnn")
- x_cnn = tf.compat.v1.placeholder(tf.float32, [None, params.patch_window, params.patch_window, params.n_channels],
- name="x_cnn")
- y = tf.compat.v1.placeholder(tf.int32, [None, 1, 1, 1], name="y")
-
- learning_rate = tf.compat.v1.placeholder_with_default(tf.constant(0.0002, dtype=tf.float32, shape=[]), shape=[],
- name="learning_rate")
- is_training_ph = tf.compat.v1.placeholder_with_default(tf.constant(False, dtype=tf.bool, shape=[]), shape=[],
- name="is_training")
- dropout = tf.compat.v1.placeholder_with_default(tf.constant(0.5, dtype=tf.float32, shape=[]), shape=[],
- name="drop_rate")
-
- pred_c1, pred_c2, pred_full, features_learnt = get_prediction(x_rnn,
- x_cnn,
- params.nunits,
- params.n_levels_lstm,
- params.nclasses,
- params.n_dims,
- params.n_timestamps)
-
- testPrediction = tf.argmax(pred_full, 1, name="prediction")
-
- loss_full = tf.compat.v1.losses.sparse_softmax_cross_entropy(labels=tf.reshape(y, [-1, 1]),
- logits=tf.reshape(pred_full, [-1, params.nclasses]))
- loss_c1 = tf.compat.v1.losses.sparse_softmax_cross_entropy(labels=tf.reshape(y, [-1, 1]),
- logits=tf.reshape(pred_c1, [-1, params.nclasses]))
- loss_c2 = tf.compat.v1.losses.sparse_softmax_cross_entropy(labels=tf.reshape(y, [-1, 1]),
- logits=tf.reshape(pred_c2, [-1, params.nclasses]))
-
- cost = loss_full + (0.3 * loss_c1) + (0.3 * loss_c2)
-
- optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate, name="optimizer").minimize(cost)
-
- correct = tf.equal(tf.argmax(pred_full, 1), tf.argmax(y, 1))
- accuracy = tf.reduce_mean(tf.cast(correct, tf.float64))
-
- # Initializer, saver, session
- init = tf.compat.v1.global_variables_initializer()
- saver = tf.compat.v1.train.Saver(max_to_keep=20)
- sess = tf.compat.v1.Session()
- sess.run(init)
-
- create_savedmodel(sess, ["x_cnn:0", "x_rnn:0", "y:0"], ["prediction:0"], params.outdir)
+# if __name__ == __main__:
+# parser = argparse.ArgumentParser()
+# parser.add_argument(
+# "--nunits",
+# type=int,
+# default=1024,
+# help="number of units"
+# )
+# parser.add_argument(
+# "--n_levels_lstm",
+# type=int,
+# default=1,
+# help="number of lstm levels"
+# )
+# parser.add_argument(
+# "--hm_epochs",
+# type=int,
+# default=400,
+# help="hm epochs"
+# )
+# parser.add_argument(
+# "--n_timestamps",
+# type=int,
+# default=37,
+# help="number of images in timeseries"
+# )
+# parser.add_argument(
+# "--n_dims",
+# type=int,
+# default=16,
+# help="number of channels in timeseries images"
+# )
+# parser.add_argument(
+# "--patch_window",
+# type=int,
+# default=25,
+# help="patch size for the high-res image"
+# )
+# parser.add_argument(
+# "--n_channels",
+# type=int,
+# default=4,
+# help="number of channels in the high-res image"
+# )
+# parser.add_argument(
+# "--nclasses",
+# type=int,
+# default=8,
+# help="number of classes"
+# )
+# parser.add_argument(
+# "--outdir",
+# help="Output directory for SavedModel",
+# required=True
+# )
+# params = parser.parse_args()
+#
+# # Create the TensorFlow graph
+# with tf.compat.v1.Graph().as_default():
+# x_rnn = tf.compat.v1.placeholder(
+# tf.float32,
+# [None, 1, 1, params.n_dims * params.n_timestamps],
+# name="x_rnn"
+# )
+# x_cnn = tf.compat.v1.placeholder(
+# tf.float32,
+# [None, params.patch_window, params.patch_window,
+# params.n_channels],
+# name="x_cnn"
+# )
+# y = tf.compat.v1.placeholder(tf.int32, [None, 1, 1, 1], name="y")
+#
+# learning_rate = tf.compat.v1.placeholder_with_default(
+# tf.constant(
+# 0.0002,
+# dtype=tf.float32,
+# shape=[]
+# ),
+# shape=[],
+# name="learning_rate"
+# )
+# is_training_ph = tf.compat.v1.placeholder_with_default(
+# tf.constant(
+# False,
+# dtype=tf.bool,
+# shape=[]
+# ),
+# shape=[],
+# name="is_training"
+# )
+# dropout = tf.compat.v1.placeholder_with_default(
+# tf.constant(
+# 0.5,
+# dtype=tf.float32,
+# shape=[]
+# ), shape=[],
+# name="drop_rate"
+# )
+#
+# pred_c1, pred_c2, pred_full, features_learnt = get_prediction(
+# x_rnn,
+# x_cnn,
+# params.nunits,
+# params.n_levels_lstm,
+# params.nclasses,
+# params.n_dims,
+# params.n_timestamps
+# )
+#
+# testPrediction = tf.argmax(pred_full, 1, name="prediction")
+#
+# loss_full = tf.compat.v1.losses.sparse_softmax_cross_entropy(
+# labels=tf.reshape(y, [-1, 1]),
+# logits=tf.reshape(pred_full, [-1, params.nclasses]))
+# loss_c1 = tf.compat.v1.losses.sparse_softmax_cross_entropy(
+# labels=tf.reshape(y, [-1, 1]),
+# logits=tf.reshape(pred_c1, [-1, params.nclasses]))
+# loss_c2 = tf.compat.v1.losses.sparse_softmax_cross_entropy(
+# labels=tf.reshape(y, [-1, 1]),
+# logits=tf.reshape(pred_c2, [-1, params.nclasses]))
+#
+# cost = loss_full + (0.3 * loss_c1) + (0.3 * loss_c2)
+#
+# optimizer = tf.compat.v1.train.AdamOptimizer(
+# learning_rate=learning_rate,
+# name="optimizer"
+# ).minimize(cost)
+#
+# correct = tf.equal(tf.argmax(pred_full, 1), tf.argmax(y, 1))
+# accuracy = tf.reduce_mean(tf.cast(correct, tf.float64))
+#
+# # Initializer, saver, session
+# init = tf.compat.v1.global_variables_initializer()
+# saver = tf.compat.v1.train.Saver(max_to_keep=20)
+# sess = tf.compat.v1.Session()
+# sess.run(init)
+#
+# create_savedmodel(
+# sess, ["x_cnn:0", "x_rnn:0", "y:0"], ["prediction:0"],
+# params.outdir
+# )
diff --git a/otbtf/examples/tensorflow_v2x/__init__.py b/otbtf/examples/tensorflow_v2x/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/__init__.py b/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
new file mode 100644
index 00000000..efd4e8ce
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
@@ -0,0 +1,92 @@
+"""
+This section contains two examples of very simple models that are not
+trainable, called **deterministic models**.
+Sometimes it can be useful to consider deterministic approaches (e.g. modeling)
+and tensorflow is a powerful numerical library that can run smoothly on many
+kind of devices such as GPUs.
+
+In this section, we will consider two deterministic models:
+
+- [L2 norm](#l2-norm): a model that computes the l2 norm of the input image
+channels, for each pixel,
+- [Scalar product](#scalar-product): a model computing the scalar product
+between two images with the same number of channels, for each pixel
+
+# L2 norm
+
+We consider a very simple model that implements the computation of the l2 norm.
+The model inputs one multispectral image (*x*), and computes the l2 norm of
+each pixel (*y*). The model is exported as a SavedModel named
+*l2_norm_savedmodel*
+
+```python
+import tensorflow as tf
+
+# Input
+x = tf.keras.Input(shape=[None, None, None], name="x") # [1, h, w, N]
+
+# Compute norm on the last axis
+y = tf.norm(x, axis=-1)
+
+# Create model
+model = tf.keras.Model(inputs={"x": x}, outputs={"y": y})
+model.save("l2_norm_savedmodel")
+```
+
+Run the code. The *l2_norm_savedmodel* file is created.
+Now run the SavedModel with `TensorflowModelServe`:
+
+```commandline
+otbcli_TensorflowModelServe \\
+-source1.il image1.tif \\
+-model.dir l2_norm_savedmodel \\
+-model.fullyconv on \\
+-out output.tif \\
+-optim.disabletiling on
+```
+
+!!! Note
+
+ As you can notice, we have set the `optim.disabletiling` to `on` which
+ disables the tiling for the processing. This means that OTB will drive the
+ regions size based on the ram value defined in OTB. We can do that safely
+ since our process has a small memory footprint, and it is not optimized
+ with tiling because it does not use any neighborhood based approach.
+ Tiling is enabled by default in `TensorflowModelServe` since it is mostly
+ intended to perform inference using 2D convolutions.
+
+# Scalar product
+
+Let's consider a simple model that inputs two multispectral image (*x1* and
+*x2*), and computes the scalar product between each pixels of the two images.
+The model is exported as a SavedModel named *scalar_product_savedmodel*
+
+```python
+import tensorflow as tf
+
+# Input
+x1 = tf.keras.Input(shape=[None, None, None], name="x1") # [1, h, w, N]
+x2 = tf.keras.Input(shape=[None, None, None], name="x2") # [1, h, w, N]
+
+# Compute scalar product
+y = tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
+
+# Create model
+model = tf.keras.Model(inputs={"x1": x1, "x2": x2}, outputs={"y": y})
+model.save("scalar_product_savedmodel")
+```
+
+Run the code. The *scalar_product_savedmodel* file is created.
+Now run the SavedModel with `TensorflowModelServe`:
+
+```commandline
+OTB_TF_NSOURCES=2 otbcli_TensorflowModelServe \\
+-source1.il image1.tif \\
+-source2.il image2.tif \\
+-model.dir scalar_product_savedmodel \\
+-model.fullyconv on \\
+-out output.tif \\
+-optim.disabletiling on # Small memory footprint, we can remove tiling
+```
+
+"""
\ No newline at end of file
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/README.md b/otbtf/examples/tensorflow_v2x/fcnn/README.md
deleted file mode 100644
index e6cfce78..00000000
--- a/otbtf/examples/tensorflow_v2x/fcnn/README.md
+++ /dev/null
@@ -1,64 +0,0 @@
-This example show how to train a small fully convolutional model using the
-OTBTF python API. In particular, the example show how a model can be trained
-(1) from **patches-images**, or (2) from **TFRecords** files.
-
-# Files
-
-- `fcnn_model.py` implements a small fully convolutional U-Net like model,
-with the preprocessing and normalization functions
-- `train_from_patches-images.py` shows how to train the model from a list of
-patches-images
-- `train_from_tfrecords.py` shows how to train the model from TFRecords files
-- `create_tfrecords.py` shows how to convert patch-images into TFRecords files
-- `helper.py` contains a few helping functions
-
-# Patches-images vs TFRecords based datasets
-
-TensorFlow datasets are the most practical way to feed a network data during
-training steps.
-In particular, they are very useful to train models with data parallelism using
-multiple workers (i.e. multiple GPU devices).
-Since OTBTF 3, two kind of approaches are available to deliver the patches:
-- Create TF datasets from **patches-images**: the first approach implemented in
-OTBTF, relying on geospatial raster formats supported by GDAL. Patches are simply
-stacked in rows. patches-images are friendly because they can be visualized
-like any other image. However this approach is **not very optimized**, since it
-generates a lot of I/O and stresses the filesystem when iterating randomly over
-patches.
-- Create TF datasets from **TFRecords** files. The principle is that a number of
-patches are stored in TFRecords files (google protubuf serialized data). This
-approach provides the best performances, since it generates less I/Os since
-multiple patches are read simultaneously together. It is the recommended approach
-to work on high end gear. It requires an additional step of converting the
-patches-images into TFRecords files.
-
-## Patches-images based datasets
-
-**Patches-images** are generated from the `PatchesExtraction` application of OTBTF.
-They consist in extracted patches stacked in rows into geospatial rasters.
-The `otbtf.DatasetFromPatchesImages` provides access to **patches-images** as a
-TF dataset. It inherits from the `otbtf.Dataset` class, which can be a base class
-to develop other raster based datasets.
-The `use_streaming` option can be used to read the patches on-the-fly
-on the filesystem. However, this can cause I/O bottleneck when one training step
-is shorter that fetching one batch of data. Typically, this is very common with
-small networks trained over large amount of data using multiple GPUs, causing the
-filesystem read operation being the weak point (and the GPUs wait for the batches
-to be ready). The class offers other functionalities, for instance changing the
-iterator class with a custom one (can inherit from `otbtf.dataset.IteratorBase`)
-which is, by default, an `otbtf.dataset.RandomIterator`. This could enable to
-control how the patches are walked, from the multiple patches-images of the
-dataset.
-
-## TFRecords batches datasets
-
-**TFRecord** based datasets are implemented in the `otbtf.tfrecords` module.
-They basically deliver patches from the TFRecords files, which can be created
-with the `to_tfrecords()` method of the `otbtf.Dataset` based classes.
-Depending on the filesystem characteristics and the computational cost of one
-training step, it can be good to select the number of samples per TFRecords file.
-Another tweak is the shuffling: since one TFRecord file contains multiple patches,
-the way TFRecords files are accessed (sometimes, we need them to be randomly
-accessed), and the way patches are accessed (within a buffer, of size set with the
-`shuffle_buffer_size`), is crucial.
-
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/__init__.py b/otbtf/examples/tensorflow_v2x/fcnn/__init__.py
new file mode 100644
index 00000000..fa80aa01
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/fcnn/__init__.py
@@ -0,0 +1,4 @@
+"""
+This module shows how to train a small fully convolutional network from
+patches extracted in images.
+"""
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 804744c2..44285d92 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -129,9 +129,12 @@ class FCNNModel(ModelBase):
def dataset_preprocessing_fn(examples: dict):
"""
Preprocessing function for the training dataset.
- This function is only used at training time, to put the data in the expected format for the training step.
- DO NOT USE THIS FUNCTION TO NORMALIZE THE INPUTS ! (see `otbtf.ModelBase.normalize_inputs` for that).
- Note that this function is not called here, but in the code that prepares the datasets.
+ This function is only used at training time, to put the data in the
+ expected format for the training step.
+ DO NOT USE THIS FUNCTION TO NORMALIZE THE INPUTS ! (see
+ `otbtf.ModelBase.normalize_inputs` for that).
+ Note that this function is not called here, but in the code that prepares
+ the datasets.
Params:
examples: dict for examples (i.e. inputs and targets stored in a single
@@ -141,15 +144,12 @@ def dataset_preprocessing_fn(examples: dict):
preprocessed examples
"""
-
- def _to_categorical(x):
- return tf.one_hot(
- tf.squeeze(tf.cast(x, tf.int32), axis=-1), depth=N_CLASSES
- )
-
return {
INPUT_NAME: examples["input_xs_patches"],
- TARGET_NAME: _to_categorical(examples["labels_patches"])
+ TARGET_NAME: tf.one_hot(
+ tf.squeeze(tf.cast(examples["labels_patches"], tf.int32), axis=-1),
+ depth=N_CLASSES
+ )
}
diff --git a/otbtf/examples/tensorflow_v2x/l2_norm.py b/otbtf/examples/tensorflow_v2x/l2_norm.py
deleted file mode 100644
index a98e9a64..00000000
--- a/otbtf/examples/tensorflow_v2x/l2_norm.py
+++ /dev/null
@@ -1,25 +0,0 @@
-"""
-This code implements a simple model that inputs one multispectral image ("x"),
-and computes the euclidean norm of each pixel ("y").
-The model is exported as a SavedModel named "l2_norm_savedmodel"
-
-To run the SavedModel:
-
-otbcli_TensorflowModelServe \
--source1.il image1.tif \
--model.dir l2_norm_savedmodel \
--model.fullyconv on \
--out output.tif \
--optim.disabletiling on # Tiling is not helping here, since its a pixel wise op.
-"""
-import tensorflow as tf
-
-# Input
-x = tf.keras.Input(shape=[None, None, None], name="x") # [1, h, w, N]
-
-# Compute norm on the last axis
-y = tf.norm(x, axis=-1)
-
-# Create model
-model = tf.keras.Model(inputs={"x": x}, outputs={"y": y})
-model.save("l2_norm_savedmodel")
diff --git a/otbtf/examples/tensorflow_v2x/scalar_product.py b/otbtf/examples/tensorflow_v2x/scalar_product.py
deleted file mode 100644
index dc67a322..00000000
--- a/otbtf/examples/tensorflow_v2x/scalar_product.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""
-This code implements a simple model that inputs two multispectral image ("x1" and "x2"),
-and computes the scalar product between each pixels of the two images.
-The model is exported as a SavedModel named "scalar_product_savedmodel"
-
-To run the SavedModel:
-
-OTB_TF_NSOURCES=2 otbcli_TensorflowModelServe \
--source1.il image1.tif \
--source2.il image2.tif \
--model.dir scalar_product_savedmodel \
--model.fullyconv on \
--out output.tif \
--optim.disabletiling on # Tiling is not helping here, since its a pixel wise op.
-"""
-import tensorflow as tf
-
-# Input
-x1 = tf.keras.Input(shape=[None, None, None], name="x1") # [1, h, w, N]
-x2 = tf.keras.Input(shape=[None, None, None], name="x2") # [1, h, w, N]
-
-# Compute scalar product
-y = tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
-
-# Create model
-model = tf.keras.Model(inputs={"x1": x1, "x2": x2}, outputs={"y": y})
-model.save("scalar_product_savedmodel")
diff --git a/otbtf/model.py b/otbtf/model.py
index 3cbc0230..c39e66ab 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -17,7 +17,9 @@
# limitations under the License.
#
# ==========================================================================*/
-""" Base class for models"""
+"""
+Base class for models.
+"""
from typing import List, Dict, Any
import abc
import logging
diff --git a/test/api_unittest.py b/test/api_unittest.py
index 29582489..2fe3fe38 100644
--- a/test/api_unittest.py
+++ b/test/api_unittest.py
@@ -1,13 +1,16 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
-import pytest
import unittest
-from test_utils import resolve_paths, files_exist, run_command_and_compare
-from otbtf.examples.tensorflow_v2x.fcnn.fcnn_model import INPUT_NAME, OUTPUT_SOFTMAX_NAME
+
+import pytest
+
+from otbtf.examples.tensorflow_v2x.fcnn import create_tfrecords
from otbtf.examples.tensorflow_v2x.fcnn import train_from_patchesimages
from otbtf.examples.tensorflow_v2x.fcnn import train_from_tfrecords
-from otbtf.examples.tensorflow_v2x.fcnn import create_tfrecords
+from otbtf.examples.tensorflow_v2x.fcnn.fcnn_model import INPUT_NAME, \
+ OUTPUT_SOFTMAX_NAME
from otbtf.model import cropped_tensor_name
+from test_utils import resolve_paths, files_exist, run_command_and_compare
INFERENCE_MAE_TOL = 10.0 # Dummy value: we don't really care of the mae value but rather the image size etc
@@ -16,21 +19,25 @@ class APITest(unittest.TestCase):
@pytest.mark.order(1)
def test_train_from_patchesimages(self):
- params = train_from_patchesimages.parser.parse_args(['--model_dir', resolve_paths('$TMPDIR/model_from_pimg'),
- '--nb_epochs', '1',
- '--train_xs',
- resolve_paths('$DATADIR/amsterdam_patches_A.tif'),
- '--train_labels',
- resolve_paths('$DATADIR/amsterdam_labels_A.tif'),
- '--valid_xs',
- resolve_paths('$DATADIR/amsterdam_patches_B.tif'),
- '--valid_labels',
- resolve_paths('$DATADIR/amsterdam_labels_B.tif')])
+ params = train_from_patchesimages.parser.parse_args([
+ '--model_dir', resolve_paths('$TMPDIR/model_from_pimg'),
+ '--nb_epochs', '1',
+ '--train_xs',
+ resolve_paths('$DATADIR/amsterdam_patches_A.tif'),
+ '--train_labels',
+ resolve_paths('$DATADIR/amsterdam_labels_A.tif'),
+ '--valid_xs',
+ resolve_paths('$DATADIR/amsterdam_patches_B.tif'),
+ '--valid_labels',
+ resolve_paths('$DATADIR/amsterdam_labels_B.tif')
+ ])
train_from_patchesimages.train(params=params)
- self.assertTrue(files_exist(['$TMPDIR/model_from_pimg/keras_metadata.pb',
- '$TMPDIR/model_from_pimg/saved_model.pb',
- '$TMPDIR/model_from_pimg/variables/variables.data-00000-of-00001',
- '$TMPDIR/model_from_pimg/variables/variables.index']))
+ self.assertTrue(files_exist([
+ '$TMPDIR/model_from_pimg/keras_metadata.pb',
+ '$TMPDIR/model_from_pimg/saved_model.pb',
+ '$TMPDIR/model_from_pimg/variables/variables.data-00000-of-00001',
+ '$TMPDIR/model_from_pimg/variables/variables.index'
+ ]))
@pytest.mark.order(2)
def test_model_inference1(self):
@@ -48,7 +55,8 @@ class APITest(unittest.TestCase):
"-output.efieldx 32 "
"-output.efieldy 32 "
"-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
- to_compare_dict={"$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
+ to_compare_dict={
+ "$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
tol=INFERENCE_MAE_TOL))
self.assertTrue(
run_command_and_compare(
@@ -64,36 +72,49 @@ class APITest(unittest.TestCase):
"-output.efieldx 64 "
"-output.efieldy 64 "
"-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
- to_compare_dict={"$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
+ to_compare_dict={
+ "$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
tol=INFERENCE_MAE_TOL))
@pytest.mark.order(3)
def test_create_tfrecords(self):
- params = create_tfrecords.parser.parse_args(['--xs', resolve_paths('$DATADIR/amsterdam_patches_A.tif'),
- '--labels', resolve_paths('$DATADIR/amsterdam_labels_A.tif'),
- '--outdir', resolve_paths('$TMPDIR/train')])
+ params = create_tfrecords.parser.parse_args([
+ '--xs', resolve_paths('$DATADIR/amsterdam_patches_A.tif'),
+ '--labels', resolve_paths('$DATADIR/amsterdam_labels_A.tif'),
+ '--outdir', resolve_paths('$TMPDIR/train')
+ ])
create_tfrecords.create_tfrecords(params=params)
- self.assertTrue(files_exist(['$TMPDIR/train/output_shapes.json',
- '$TMPDIR/train/output_types.json',
- '$TMPDIR/train/0.records']))
- params = create_tfrecords.parser.parse_args(['--xs', resolve_paths('$DATADIR/amsterdam_patches_B.tif'),
- '--labels', resolve_paths('$DATADIR/amsterdam_labels_B.tif'),
- '--outdir', resolve_paths('$TMPDIR/valid')])
+ self.assertTrue(files_exist([
+ '$TMPDIR/train/output_shapes.json',
+ '$TMPDIR/train/output_types.json',
+ '$TMPDIR/train/0.records'
+ ]))
+ params = create_tfrecords.parser.parse_args([
+ '--xs', resolve_paths('$DATADIR/amsterdam_patches_B.tif'),
+ '--labels', resolve_paths('$DATADIR/amsterdam_labels_B.tif'),
+ '--outdir', resolve_paths('$TMPDIR/valid')
+ ])
create_tfrecords.create_tfrecords(params=params)
- self.assertTrue(files_exist(['$TMPDIR/valid/output_shapes.json',
- '$TMPDIR/valid/output_types.json',
- '$TMPDIR/valid/0.records']))
+ self.assertTrue(files_exist([
+ '$TMPDIR/valid/output_shapes.json',
+ '$TMPDIR/valid/output_types.json',
+ '$TMPDIR/valid/0.records'
+ ]))
@pytest.mark.order(4)
def test_train_from_tfrecords(self):
- params = train_from_tfrecords.parser.parse_args(['--model_dir', resolve_paths('$TMPDIR/model_from_tfrecs'),
- '--nb_epochs', '1',
- '--tfrecords_dir', resolve_paths('$TMPDIR')])
+ params = train_from_tfrecords.parser.parse_args([
+ '--model_dir', resolve_paths('$TMPDIR/model_from_tfrecs'),
+ '--nb_epochs', '1',
+ '--tfrecords_dir', resolve_paths('$TMPDIR')
+ ])
train_from_tfrecords.train(params=params)
- self.assertTrue(files_exist(['$TMPDIR/model_from_tfrecs/keras_metadata.pb',
- '$TMPDIR/model_from_tfrecs/saved_model.pb',
- '$TMPDIR/model_from_tfrecs/variables/variables.data-00000-of-00001',
- '$TMPDIR/model_from_tfrecs/variables/variables.index']))
+ self.assertTrue(files_exist([
+ '$TMPDIR/model_from_tfrecs/keras_metadata.pb',
+ '$TMPDIR/model_from_tfrecs/saved_model.pb',
+ '$TMPDIR/model_from_tfrecs/variables/variables.data-00000-of-00001',
+ '$TMPDIR/model_from_tfrecs/variables/variables.index'
+ ]))
@pytest.mark.order(5)
def test_model_inference2(self):
@@ -111,7 +132,10 @@ class APITest(unittest.TestCase):
"-output.efieldx 32 "
"-output.efieldy 32 "
"-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
- to_compare_dict={"$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
+ to_compare_dict={
+ "$DATADIR/classif_model4_softmax.tif":
+ "$TMPDIR/classif_model4_softmax.tif"
+ },
tol=INFERENCE_MAE_TOL))
self.assertTrue(
@@ -128,7 +152,10 @@ class APITest(unittest.TestCase):
"-output.efieldx 64 "
"-output.efieldy 64 "
"-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
- to_compare_dict={"$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
+ to_compare_dict={
+ "$DATADIR/classif_model4_softmax.tif":
+ "$TMPDIR/classif_model4_softmax.tif"
+ },
tol=INFERENCE_MAE_TOL))
--
GitLab
From 5ca163604459220984606b91c389746be285cb93 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 29 Mar 2023 19:54:54 +0200
Subject: [PATCH 03/72] DOC: wip
---
doc/api_model_generalities.md | 11 +-
doc/api_tutorial.md | 243 ++++++++++++++----
doc/app_inference.md | 29 ++-
doc/app_sampling.md | 2 +-
doc/app_training.md | 9 +-
doc/build_from_sources.md | 7 +-
doc/doc_requirements.txt | 1 +
doc/docker_build.md | 23 --
doc/docker_troubleshooting.md | 26 +-
doc/docker_use.md | 4 +-
mkdocs.yml | 12 +-
otbtf/dataset.py | 2 +
otbtf/examples/tensorflow_v1x/__init__.py | 22 +-
.../tensorflow_v2x/deterministic/__init__.py | 2 +
.../tensorflow_v2x/deterministic/l2_norm.py | 24 ++
.../deterministic/scalar_prod.py | 26 ++
.../examples/tensorflow_v2x/fcnn/__init__.py | 2 +-
otbtf/model.py | 4 +-
otbtf/tfrecords.py | 2 +
otbtf/utils.py | 2 +
20 files changed, 348 insertions(+), 105 deletions(-)
create mode 100644 otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py
create mode 100644 otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py
diff --git a/doc/api_model_generalities.md b/doc/api_model_generalities.md
index e0c0f42a..a4e4a640 100644
--- a/doc/api_model_generalities.md
+++ b/doc/api_model_generalities.md
@@ -14,13 +14,14 @@ performs the inference of several patches simultaneously. In
**fully-convolutional mode**, a single slice of the batch dimension is used.
- **dim 1** and **2** are for the spatial dimensions,
- **dim 3** is for the image channels. Even if your image have only 1 channel,
-you must set a shape value equals to 1 for the last dimension of the input placeholder.
+you must set a shape value equals to 1 for the last dimension of the input
+placeholder.
## Inputs shapes
-For nets intended to work in **patch-based** mode, you can stick with a
-placeholder where you define your patch size explicitly in **dim 1** and
-**dim 2**. However, for nets intended to work in **fully-convolutional** mode,
+For networks intended to work in **patch-based** mode, you can stick with a
+placeholder having a patch size explicitly defined in **dims 1 and 2**.
+However, for networks intended to work in **fully-convolutional** mode,
you must set `None` in **dim 1** and **dim 2** (before Tensorflow 2.X, it was
possible to feed placeholders with a tensor of different size where the dims
were defined). For instance, let consider an input raster with 4 spectral
@@ -28,7 +29,7 @@ bands: the input shape of the model input would be like `[None, None, None, 4]`
to work in fully-convolutional mode. By doing so, the use of input images of
any size is enabled (`TensorflowModelServe` will automatically compute the
input/output regions sizes to process, given the **receptive field** and
-**expression field** of your net).
+**expression field** of your network).
## Outputs dimensions
diff --git a/doc/api_tutorial.md b/doc/api_tutorial.md
index b4832b7a..b4787597 100644
--- a/doc/api_tutorial.md
+++ b/doc/api_tutorial.md
@@ -1,13 +1,14 @@
# Build and train deep learning models
-This section covers the use of the Python API.
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/examples/tensorflow_v2x/fcnn){ .md-button }
+
+This section covers the use of the Python API with deep learnig models.
It shows how to build and train a small fully convolutional model from patches
-extracted in the images. The example show how a model can be trained (1) from
+extracted in the images. The example shows how a model can be trained (1) from
**patches-images**, or (2) from **TFRecords** files.
## Classes and files
-All files are located in *otbtf/examples/tensorflow_v2x*.
- **fcnn_model.py** implements a small fully convolutional U-Net like model,
called `FCNNModel`, with the preprocessing and normalization functions that
inherit from `otbtf.BaseModel`
@@ -25,6 +26,7 @@ training steps.
In particular, they are very useful to train models with data parallelism using
multiple workers (i.e. multiple GPU devices).
Since OTBTF 3, two kind of approaches are available to deliver the patches:
+
- Create TF datasets from **patches-images**: the first approach implemented in
OTBTF, relying on geospatial raster formats supported by GDAL. Patches are
stacked in rows. patches-images are friendly because they can be visualized
@@ -58,8 +60,10 @@ dataset.
Suppose you have extracted some patches with the `PatchesExtraction`
application with 2 sources:
+
- Source "xs": patches images *xs_1.tif*, ..., *xs_N.tif*
- Source "labels": patches images *labels_1.tif*, ..., *labels_N.tif*
+
To create a dataset from this set of patches can be done with
`otbtf.DatasetFromPatchesImages` as shown below.
@@ -117,6 +121,8 @@ tf_dataset = dataset.read(
## Model
+### Overview
+
Let's define the setting for our model:
```python
@@ -133,10 +139,16 @@ TARGET_NAME = "predictions"
OUTPUT_SOFTMAX_NAME = "predictions_softmax_tensor"
```
-Now we want to implement our model. Of course we can build from scratch
-a model based on `tf.keras.Model`, but we will see how OTBTF helps a lot with
-`otbtf.BaseModel`.
-Let's take a look to this schema:
+Our model estimates *2* classes. The input name is *input_xs*, and the
+target output is *predictions*. This target output will be used to compute
+the loss value, which is used ultimately to drive the learning of the
+network. The name of the output that we want to use at inference time is
+*predictions_softmax_tensor*. We won't use this tensor for anything else than
+inference.
+
+To build our model, we can build from scratch building on `tf.keras.Model`,
+but we will see how OTBTF helps a lot with the `otbtf.BaseModel` class.
+First, let's take a look to this schema:
@@ -145,9 +157,11 @@ As we can see, we can distinguish two main functional blocks:
- training
- inference
+### Dataset transformation
+
During training, we need to preprocess the samples generated by the dataset to
feed the network and the loss computation, that will guide how weights will be
-updated. This **data preprocessing** is generally required to format the data
+updated. This **data transformation** is generally required to put the data
in the format expected by the model.
@@ -168,6 +182,11 @@ def dataset_preprocessing_fn(examples: dict):
}
```
+As you can see, we don't modify the input tensor, since we want to use it
+as it in the model.
+
+### Model inputs preprocessing
+
The model is intended to work on real world images, which have often 16 bits
signed integers as pixel values. The model has to normalize these values such
as they fit the [0, 1] range before applying the convolutions. This is called
@@ -188,6 +207,8 @@ class FCNNModel(ModelBase):
return {INPUT_NAME: tf.cast(inputs[INPUT_NAME], tf.float32) * 0.0001}
```
+### Network implementation
+
Then we implement the model itself in `FCNNModel.get_outputs()`. The model
must return a dictionary of tensors. All keys of the target tensors must be in
the returned dictionary (in our case: the *predictions* tensor). These target
@@ -258,7 +279,7 @@ First we declare the strategy used. Here we chose
computing resource.
```python
- strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
+strategy = tf.distribute.MirroredStrategy()
```
Then we instantiate, compile, and train the model within the `strategy` scope.
@@ -266,24 +287,10 @@ Then we instantiate, compile, and train the model within the `strategy` scope.
First, we create an instance of our model:
```python
- with strategy.scope():
- model = FCNNModel(dataset_element_spec=ds_train.element_spec)
+with strategy.scope():
+ model = FCNNModel(dataset_element_spec=ds_train.element_spec)
```
-!!! Info
-
- As you can see, we just have provided `dataset_element_spec` to the model
- initialization. If you take a look to
- [`ModelBase.__init__()`](reference/otbtf/model.html#otbtf.model.ModelBase.__init__)
- you can notice the `inference_cropping` parameter, with the default values
- set to [16, 32, 64, 96, 128]. What is the use of these values?
- Take another look in
- [`ModelBase.postprocess_outputs()`](reference/otbtf/model.html#otbtf.model.ModelBase.postprocess_outputs).
- The model will create an array of output, each one cropped to one value of
- `inference_cropping`. These cropped output enable to avoid or lower the
- magnitude of the blocking artifacts in convolutional models, caused by the
- convolutions.
-
In all the following, we are still inside the `strategy` scope.
After the model is instantiated, we compile it using:
@@ -297,17 +304,17 @@ our model
dataset
```python
- model.compile(
- loss=tf.keras.losses.CategoricalCrossentropy(),
- optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),
- metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()]
- )
+ model.compile(
+ loss=tf.keras.losses.CategoricalCrossentropy(),
+ optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),
+ metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()]
+ )
```
We can then train our model using Keras:
```python
- model.fit(ds_train, epochs=100, validation_data=ds_valid)
+ model.fit(ds_train, epochs=100, validation_data=ds_valid)
```
At the end of the training (here we just perform 100 epochs over the training
@@ -315,43 +322,185 @@ dataset, then stop), we could perform some evaluation over an additional test
dataset:
```python
- model.evaluate(ds_test, batch_size=64)
+ model.evaluate(ds_test, batch_size=64)
```
Finally we can save our model as a SavedModel:
```python
- model.save("/tmp/my_1st_savedmodel")
+ model.save("/tmp/my_1st_savedmodel")
```
The whole code is available in the *otbtf/examples/tensorflow_v2x/fcnn*
directory.
+[`ModelBase.__init__()`](reference/otbtf/examples/tensorflow_v2x/fcnn/__init__.html)
## Inference
+This section show how to apply the fully convolutional model over an entire
+image.
+
+### Postprocessing to avoid blocking artifacts
+
+The class `otbtf.ModelBase` provides the necessary to enable fully
+convolutional models to be applied over large images, avoiding blocking
+artifacts caused by convolutions at the borders of tensors.
+`ModelBase` comes with a `postprocess_outputs()`, that process the outputs
+tensors returned by `get_outputs()`. This creates new outputs, aiming to be
+used at inference time. The default implementation of
+`ModelBase.postprocess_outputs()` avoids blocking artifacts, by keeping
+only the values of the central part of the tensors in spatial dimensions (you
+can read more on the subject in this
+[book](https://doi.org/10.1201/9781003020851)).
+
+If you take a look to
+[`ModelBase.__init__()`](reference/otbtf/model.html#otbtf.model.ModelBase.__init__)
+you can notice the `inference_cropping` parameter, with the default values
+set to [16, 32, 64, 96, 128]. Now if you take another look in
+[`ModelBase.postprocess_outputs()`](reference/otbtf/model.html#otbtf.model.ModelBase.postprocess_outputs),
+you can see how these values are used: the model will create an array of
+outputs, each one cropped to one value of `inference_cropping`. These cropped
+outputs enable to avoid or lower the magnitude of the blocking artifacts
+in convolutional models.
+The new outputs tensors are named by the
+[`cropped_tensor_name()`](reference/otbtf/model.html#otbtf.model.cropped_tensor_name)
+function, that returns a new name corresponding to:
+```python
+f"{tensor_name}_crop{crop}"
+```
+For instance, for the new output tensor created for *predictions*, that
+removes 32 pixels from the borders in the spatial dimensions, would be named
+***predictions_crop32***.
+
+### How to choose the right cropping value?
+
+Theoretically, we can determine the part of the output image that is not
+polluted by the convolutional padding.
+For a 2D convolution of stride \(s\) and kernel size \(k\), we can deduce the
+valid output size \(y\) from input size \(x\) using this expression:
+$$
+y = \left[\frac{x - k + 1}{s}\right]
+$$
+For a 2D transposed convolution of stride \(s\) and kernel size \(k\), we can
+deduce the valid output size \(y\) from input size \(x\) using this expression:
+$$
+y = (x * s) - k + 1
+$$
+
+Let's consider a chunk of input image of size 128, and check the valid output
+size of our model:
+
+| Conv. name | Conv. type | Kernel | Stride | Out. size | Valid out. size |
+|------------|-------------------|--------|--------|-----------|-----------------|
+| *conv1* | Conv2D | 3 | 2 | 64 | 63 |
+| *conv2* | Conv2D | 3 | 2 | 32 | 30 |
+| *conv3* | Conv2D | 3 | 2 | 16 | 14 |
+| *conv4* | Conv2D | 3 | 2 | 8 | 6 |
+| *tconv1* | Transposed Conv2D | 3 | 2 | 16 | 10 |
+| *tconv2* | Transposed Conv2D | 3 | 2 | 32 | 18 |
+| *tconv3* | Transposed Conv2D | 3 | 2 | 64 | 34 |
+
+This shows that our model can be applied in a fully convolutional fashion
+without generating blocking artifacts, using the central part of the output of
+size 34. This is equivalent to remove \((128 - 24)/2 = 47\) pixels from
+the borders of the output. We can hence use the output cropped with **64**
+pixels, named ***predictions_crop64***.
+
+!!! Info
+
+ Very deep networks will lead to very large cropping values.
+ In these cases, there is a tradeoff between numerical
+ exactness VS computational cost. In practice, expression field can be
+ ridiculously enlarged since most of the networks learn to disminish the
+ convolutional distortion at the border of the training patches.
+
+### TensorflowModelServe parameters
+
We can use the exported SavedModel, located in */tmp/my_1st_savedmodel*, using
either:
+
- The OTB command line interface,
- The OTB Python wrapper,
- The PyOTB Python wrapper,
- The OTB Graphical User Interface,
-- If we have configured QGIS with the descriptors of OTBTF applications, we
-also could have used QGIS for that.
+- QGIS (you have to copy the descriptors of OTBTF applications in QGIS
+configuration folder).
+In the following, we focus only the CLI and python.
+
+In the following subsections, we run `TensorflowModelServe` over the input
+image, with the following parameters:
+
+- the input name is ***input_xs***
+- the output name is ***predictions_crop64*** (cropping margin of 64 pixels)
+- we choose a receptive field of ***256*** and an expression field of
+***128*** so that they match the cropping margin of 64 pixels.
-In the following, we focus only the CLI and python ways.
### Command Line Interface
+Open a terminal and run the following command:
+
```commandline
-otbcli_TensorflowModelServe \\
--source1.il $DATADIR/fake_spot6.jp2 \\
--source1.rfieldx 128 \\
--source1.rfieldy 128 \\
--source1.placeholder "input_xs" \\
--model.dir /tmp/my_1st_savedmodel \\
--model.fullyconv on \\
--output.names "predictions_crop32" \\
--output.efieldx 64 \\
--output.efieldy 64 \\
+otbcli_TensorflowModelServe \
+-source1.il $DATADIR/fake_spot6.jp2 \
+-source1.rfieldx 256 \
+-source1.rfieldy 256 \
+-source1.placeholder "input_xs" \
+-model.dir /tmp/my_1st_savedmodel \
+-model.fullyconv on \
+-output.names "predictions_crop64" \
+-output.efieldx 128 \
+-output.efieldy 128 \
-out softmax.tif
-```
\ No newline at end of file
+```
+
+### OTB Python wrapper
+
+The previous command translates in the following in python, using the OTB
+python wrapper:
+
+```python
+import otbApplication
+app = otbApplication.Registry.CreateApplication("TensorflowModelServe")
+app.SetParameterStringList("source1.il", ["fake_spot6.jp2"])
+app.SetParameterInt("source1.rfieldx", 256)
+app.SetParameterInt("source1.rfieldy", 256)
+app.SetParameterString("source1.placeholder", "input_xs")
+app.SetParameterString("model.dir", "/tmp/my_1st_savedmodel")
+app.EnableParameter("fullyconv")
+app.SetParameterStringList("output.names", ["predictions_crop64"])
+app.SetParameterInt("output.efieldx", 128)
+app.SetParameterInt("output.efieldy", 128)
+app.SetParameterString("out", "softmax.tif")
+app.ExecuteAndWriteOutput()
+```
+
+### PyOTB
+
+Using PyOTB is nicer:
+
+```python
+import pyotb
+pyotb.TensorflowModelServe({
+ "source1.il": "fake_spot6.jp2",
+ "source1.rfieldx": 256,
+ "source1.rfieldy": 256,
+ "source1.placeholder": "input_xs",
+ "model.dir": "/tmp/my_1st_savedmodel",
+ "fullyconv": True,
+ "output.names": ["predictions_crop64"],
+ "output.efieldx": 128,
+ "output.efieldy": 128,
+ "out": "softmax.tif",
+})
+```
+
+!!! Note
+
+ The processing can be optimized using the `optim` parameters group.
+ In a terminal, type `otbcli_TensorflowModelServe --help optim` for more
+ information. Also, the extended filenames of the orfeo toolbox enables to
+ control the output image chunk size and tiling/stripping layout. Combined
+ with the `optim` parameters, you will likely always find the best settings
+ suited for the hardware. Also, the receptive and expression fields sizes
+ have a major contribution.
\ No newline at end of file
diff --git a/doc/app_inference.md b/doc/app_inference.md
index 9b1f8840..1e369e03 100644
--- a/doc/app_inference.md
+++ b/doc/app_inference.md
@@ -67,8 +67,21 @@ As explained, the **scale factor** provided for the
_output tensors_ is related to this _reference input source_.
The user can ask for multiple _output tensors_, that will be stack along the
channel dimension of the output raster.
-However, if the sizes of those _output tensors_ are not consistent (e.g. a
-different number of (x,y) elements), an exception will be thrown.
+
+!!! Warning
+
+ Multiple outputs names can be provided which results in stacked tensors in
+ the output image along the channels dimension. In this case, tensors must
+ have the same size in spatial dimension: if the sizes of _output tensors_
+ are not consistent (e.g. a different number of (x,y) elements), an
+ exception will be thrown.
+
+!!! Warning
+
+ If no output tensor name is specified, the application will try to grab
+ the first output tensor found in the SavedModel. This is okay with models
+ having a single output (see
+ [deterministic models section](reference/otbtf/examples/tensorflow_v2x/deterministic/__init__.html)).
@@ -92,12 +105,16 @@ The application description can be displayed using:
otbcli_ImageClassifierFromDeepFeatures --help
```
-Note that you can still set the `OTB_TF_NSOURCES` environment variable.
+!!! Note
+
+ You can still set the `OTB_TF_NSOURCES` environment variable to change the
+ number of sources.
-# Example
+## Example
-We assume that we have already followed the *training* section, and we start
-from the files generated at the end of the training.
+We assume that we have already followed the
+[*training* section](app_training.html). We start from the files generated at
+the end of the training step.
After this step, we use the trained model to produce the entire map of forest
over the whole Spot-7 image.
diff --git a/doc/app_sampling.md b/doc/app_sampling.md
index 480ff5e0..0d09037a 100644
--- a/doc/app_sampling.md
+++ b/doc/app_sampling.md
@@ -37,7 +37,7 @@ The application description can be displayed using:
otbcli_PatchesSelection --help
```
-## PatchesExtraction.
+## PatchesExtraction
The `PatchesExtraction` application performs the extraction of patches in
images from the following:
diff --git a/doc/app_training.md b/doc/app_training.md
index dd9fd2c8..30dacb10 100644
--- a/doc/app_training.md
+++ b/doc/app_training.md
@@ -5,7 +5,7 @@
v1. To have a full control over the model implementation and training
process, the Tensorflow API v2 with Keras is the way to go.
-# TensorflowModelTrain
+## TensorflowModelTrain
Here we assume that you have produced patches using the **PatchesExtraction**
application, and that you have a **SavedModel** stored in a directory somewhere
@@ -72,10 +72,11 @@ The application description can be displayed using:
otbcli_TrainClassifierFromDeepFeatures --help
```
-# Example
+## Example
-We assume that we have already followed the *sampling* section, and we start
-from the files generated at the end of the patches extraction.
+We assume that we have already followed the
+[*sampling* section](app_sampling.html). We start from the files generated at
+the end of the patches extraction.
Now we have two images for patches and labels.
We can split them to distinguish test/validation groups (with the **ExtractROI**
diff --git a/doc/build_from_sources.md b/doc/build_from_sources.md
index 4189db1b..e97d46cb 100644
--- a/doc/build_from_sources.md
+++ b/doc/build_from_sources.md
@@ -1,6 +1,5 @@
-# How to build OTBTF from sources
+# Build OTBTF from sources
-Building OTBTF from sources is the hard way.
These instructions explain how to build on Ubuntu 18 with last CUDA
drivers, TensorFlow r2.1 and OTB 7.1.0.
@@ -135,8 +134,8 @@ your CPU (For instance here is AVX, AVX2, FMA, SSE4.1, SSE4.2 that play fine
on a modern intel CPU). You have to tell Bazel to build:
1. The TensorFlow python pip package
- 2. The libtensorflow_cc.so library
- 3. The libtensorflow_framework.so library
+ 2. The *libtensorflow_cc.so* library
+ 3. The *libtensorflow_framework.so* library
```
bazel build -c opt --copt=-mavx --copt=-mavx2 --copt=-mfma \
diff --git a/doc/doc_requirements.txt b/doc/doc_requirements.txt
index 4f596924..e5740205 100644
--- a/doc/doc_requirements.txt
+++ b/doc/doc_requirements.txt
@@ -5,3 +5,4 @@ mkdocs-gen-files
mkdocs-section-index
mkdocs-literate-nav
mkdocs-mermaid2-plugin
+python-markdown-math
diff --git a/doc/docker_build.md b/doc/docker_build.md
index 61abc4c1..b2a32ec0 100644
--- a/doc/docker_build.md
+++ b/doc/docker_build.md
@@ -212,26 +212,3 @@ docker create -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$DISPLAY -it \
docker start -i otbtf-gui
$ mapla
```
-
-## Common errors
-
-Build :
-
-```
-Error response from daemon:
-manifest for nvidia/cuda:11.0-cudnn8-devel-ubuntu20.04 not found:
-manifest unknown: manifest unknown
-```
-
-=> Image is missing from dockerhub
-
-Run :
-
-```
-failed call to cuInit:
-UNKNOWN ERROR (303) / no NVIDIA GPU device is present:
-/dev/nvidia0 does not exist
-```
-
-=> Nvidia driver is missing or disabled, make sure to add
-` --gpus=all` to your docker run or create command
diff --git a/doc/docker_troubleshooting.md b/doc/docker_troubleshooting.md
index 5835a6b9..c34b7d2d 100644
--- a/doc/docker_troubleshooting.md
+++ b/doc/docker_troubleshooting.md
@@ -4,7 +4,31 @@ You can find plenty of help on the web about docker.
This section only provides the basics for newcomers that are eager to use
OTBTF!
This section is largely inspired from the
-[moringa docker help](https://gitlab.irstea.fr/raffaele.gaetano/moringa/blob/develop/docker/README.md). Big thanks to them.
+[moringa docker help](https://gitlab.irstea.fr/raffaele.gaetano/moringa/blob/develop/docker/README.md).
+Big thanks to the authors.
+
+## Common errors
+
+### Manifest unknown
+
+```
+Error response from daemon:
+manifest for nvidia/cuda:11.0-cudnn8-devel-ubuntu20.04 not found:
+manifest unknown: manifest unknown
+```
+
+This means that the docker image is missing from dockerhub.
+
+### failed call to cuInit
+
+```
+failed call to cuInit:
+UNKNOWN ERROR (303) / no NVIDIA GPU device is present:
+/dev/nvidia0 does not exist
+```
+
+Nvidia driver is missing or disabled, make sure to add
+` --gpus=all` to your docker run or create command
## Useful diagnostic commands
diff --git a/doc/docker_use.md b/doc/docker_use.md
index 59990bef..78d6baa9 100644
--- a/doc/docker_use.md
+++ b/doc/docker_use.md
@@ -18,7 +18,8 @@ Since OTBTF >= 3.2.1 you can find latest docker images on
The list of older releases is available [here](#older-images).
-!!! warning
+!!! Warning
+
Until r2.4, all images are development-ready, and the sources are located
in `/work/`.
Since r2.4, development-ready images have the source in `/src/` and are
@@ -76,6 +77,7 @@ See here how to install docker on Ubuntu
Troubleshooting:
+
- [Docker for windows WSL documentation](https://docs.docker.com/docker-for-windows/wsl)
- [WSL2 installation steps](https://docs.microsoft.com/en-us/windows/wsl/install-win10)
diff --git a/mkdocs.yml b/mkdocs.yml
index c1603ff8..be9f396a 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -44,12 +44,11 @@ nav:
- Model generalities: api_model_generalities.md
- Deterministic models: reference/otbtf/examples/tensorflow_v2x/deterministic/__init__.md
- Build and train deep learning models: api_tutorial.md
-- Tensorflow API v1 (deprecated):
- - A few deep learning models: reference/otbtf/examples/tensorflow_v1x/__init__.md
- Python API references:
- dataset: reference/otbtf/dataset.md
- tfrecords: reference/otbtf/tfrecords.md
- model: reference/otbtf/model.md
+- Tensorflow API v1: reference/otbtf/examples/tensorflow_v1x/__init__.md
# Customization
extra:
@@ -61,6 +60,7 @@ extra:
use_directory_urls: false # this creates some otbtf/core.html pages instead of otbtf/core/index.html
markdown_extensions:
+ - attr_list
- admonition
- toc:
permalink: true
@@ -72,9 +72,15 @@ markdown_extensions:
- pymdownx.snippets
- pymdownx.details
- pymdownx.superfences
-
+ - mdx_math
+ - pymdownx.emoji:
+ emoji_index: !!python/name:materialx.emoji.twemoji
+ emoji_generator: !!python/name:materialx.emoji.to_svg
+
# Rest of the navigation..
site_name: "OTBTF"
repo_url: https://github.com/remicres/otbtf
repo_name: otbtf
docs_dir: doc/
+extra_javascript:
+ - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.4/MathJax.js?config=TeX-AMS-MML_HTMLorMML
diff --git a/otbtf/dataset.py b/otbtf/dataset.py
index 4833cb00..ae4e7d6a 100644
--- a/otbtf/dataset.py
+++ b/otbtf/dataset.py
@@ -18,6 +18,8 @@
#
# ==========================================================================*/
"""
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/dataset.py){ .md-button }
+
Contains stuff to help working with TensorFlow and geospatial data in the
OTBTF framework.
"""
diff --git a/otbtf/examples/tensorflow_v1x/__init__.py b/otbtf/examples/tensorflow_v1x/__init__.py
index 2ed0ba3a..5e885494 100644
--- a/otbtf/examples/tensorflow_v1x/__init__.py
+++ b/otbtf/examples/tensorflow_v1x/__init__.py
@@ -1,21 +1,27 @@
"""
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/examples/tensorflow_v1x/){ .md-button }
+
This section provides some examples of ready-to-use deep learning
-architectures built with the TensorFlow API from python. All models used are
-provided in the `otbtf/examples/tensorflow_v1x/` directory.
+architectures built with the TensorFlow API v1 from python.
+
+!!! warning
+
+ This section is no longer maintained. We recommend to build and train
+ models using Keras, which has become the primary interface for Tensorflow
+ after version 2.0.
+
+ `tricks.py` is here for backward compatibility with codes based on OTBTF 1.x
+ and 2.x.
!!! note
While we recommend to use the Tensorflow API v2, the following examples are
- still great to understand how the models work, particulary in inference
- mode. Models based on the TF API v1 can be trained easily with the
+ still great to understand how the models work, in particular in inference
+ mode. Models based on the Tensorflow API v1 can be trained easily with the
`TensorflowModelTrain` application, which allows beginners to focus more
on the overall process rather than the code.
-!!! warning
-
- `tricks.py` is here for backward compatibility with codes based on OTBTF 1.x
- and 2.x.
**Table of Contents**
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/__init__.py b/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
index efd4e8ce..f84e2e75 100644
--- a/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
+++ b/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
@@ -1,4 +1,6 @@
"""
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/examples/tensorflow_v2x/deterministic){ .md-button }
+
This section contains two examples of very simple models that are not
trainable, called **deterministic models**.
Sometimes it can be useful to consider deterministic approaches (e.g. modeling)
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py b/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py
new file mode 100644
index 00000000..a2dd7136
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py
@@ -0,0 +1,24 @@
+"""
+This code implements a simple model that inputs one multispectral image ("x"),
+and computes the euclidean norm of each pixel ("y").
+The model is exported as a SavedModel named "l2_norm_savedmodel"
+To run the SavedModel:
+otbcli_TensorflowModelServe \
+-source1.il image1.tif \
+-model.dir l2_norm_savedmodel \
+-model.fullyconv on \
+-out output.tif \
+-optim.disabletiling on # Tiling is not helping here, since its a pixel wise op.
+
+"""
+import tensorflow as tf
+
+# Input
+x = tf.keras.Input(shape=[None, None, None], name="x") # [1, h, w, N]
+
+# Compute norm on the last axis
+y = tf.norm(x, axis=-1)
+
+# Create model
+model = tf.keras.Model(inputs={"x": x}, outputs={"y": y})
+model.save("l2_norm_savedmodel")
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py b/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py
new file mode 100644
index 00000000..3f41e891
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py
@@ -0,0 +1,26 @@
+"""
+This code implements a simple model that inputs two multispectral image ("x1" and "x2"),
+and computes the scalar product between each pixels of the two images.
+The model is exported as a SavedModel named "scalar_product_savedmodel"
+To run the SavedModel:
+OTB_TF_NSOURCES=2 otbcli_TensorflowModelServe \
+-source1.il image1.tif \
+-source2.il image2.tif \
+-model.dir scalar_product_savedmodel \
+-model.fullyconv on \
+-out output.tif \
+-optim.disabletiling on # Tiling is not helping here, since its a pixel wise op.
+"""
+import tensorflow as tf
+
+# Input
+x1 = tf.keras.Input(shape=[None, None, None], name="x1") # [1, h, w, N]
+x2 = tf.keras.Input(shape=[None, None, None], name="x2") # [1, h, w, N]
+
+# Compute scalar product
+y = tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
+
+# Create model
+model = tf.keras.Model(inputs={"x1": x1, "x2": x2}, outputs={"y": y})
+model.save("scalar_product_savedmodel")
+
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/__init__.py b/otbtf/examples/tensorflow_v2x/fcnn/__init__.py
index fa80aa01..7aa1b166 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/__init__.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/__init__.py
@@ -1,4 +1,4 @@
"""
-This module shows how to train a small fully convolutional network from
+Example showing how to work with a small fully convolutional network from
patches extracted in images.
"""
diff --git a/otbtf/model.py b/otbtf/model.py
index c39e66ab..554782b7 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -18,6 +18,8 @@
#
# ==========================================================================*/
"""
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/model.py){ .md-button }
+
Base class for models.
"""
from typing import List, Dict, Any
@@ -301,4 +303,4 @@ def cropped_tensor_name(tensor_name: str, crop: int):
name for the cropped tensor
"""
- return "{}_crop{}".format(tensor_name, crop)
+ return f"{tensor_name}_crop{crop}"
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index af0a0bd1..4fe1173c 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -18,6 +18,8 @@
#
# ==========================================================================*/
"""
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/tfrecords.py){ .md-button }
+
The tfrecords module provides an implementation for the TFRecords files
read/write
"""
diff --git a/otbtf/utils.py b/otbtf/utils.py
index c2b33084..f2881df5 100644
--- a/otbtf/utils.py
+++ b/otbtf/utils.py
@@ -18,6 +18,8 @@
#
# ==========================================================================*/
"""
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/utils.py){ .md-button }
+
The utils module provides some helpers to read patches using gdal
"""
from osgeo import gdal
--
GitLab
From ec3b66941bdfadc9bfbcfb0a64cb825030fc0000 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 29 Mar 2023 21:31:12 +0200
Subject: [PATCH 04/72] CI: use pip instead apt for flake8, pylint, codespell
---
.gitlab-ci.yml | 14 ++---
doc/app_overview.md | 4 +-
otbtf/__init__.py | 15 +++---
otbtf/dataset.py | 53 ++++++++-----------
.../tensorflow_v2x/deterministic/__init__.py | 5 +-
.../tensorflow_v2x/deterministic/l2_norm.py | 13 +++--
.../deterministic/scalar_prod.py | 20 ++++---
.../tensorflow_v2x/fcnn/create_tfrecords.py | 7 ++-
otbtf/examples/tensorflow_v2x/fcnn/helper.py | 8 +--
.../fcnn/train_from_patchesimages.py | 15 ++++--
.../fcnn/train_from_tfrecords.py | 4 ++
otbtf/model.py | 17 ++++--
otbtf/tfrecords.py | 18 +++----
otbtf/utils.py | 5 +-
14 files changed, 109 insertions(+), 89 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a03ca2ca..b4abcc6c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -95,20 +95,22 @@ docker image:
flake8:
extends: .static_analysis_base
script:
- - sudo apt update && sudo apt install flake8 -y
- - python -m flake8 --max-line-length=120 --per-file-ignores="__init__.py:F401" $OTBTF_SRC/otbtf
+ - sudo pip install flake8
+ - flake8 $OTBTF_SRC/otbtf
pylint:
extends: .static_analysis_base
script:
- - sudo apt update && sudo apt install pylint -y
- - pylint --logging-format-style=old --disable=too-many-nested-blocks,too-many-locals,too-many-statements,too-few-public-methods,too-many-instance-attributes,too-many-arguments --ignored-modules=tensorflow --max-line-length=120 --logging-format-style=new $OTBTF_SRC/otbtf
+ - sudo pip install pylint
+ - pylint $OTBTF_SRC/otbtf
codespell:
extends: .static_analysis_base
script:
- - sudo pip install codespell && codespell
-
+ - sudo pip install codespell
+ - codespell otbtf
+ - codespell doc
+
cppcheck:
extends: .static_analysis_base
script:
diff --git a/doc/app_overview.md b/doc/app_overview.md
index b782e781..7961ca04 100644
--- a/doc/app_overview.md
+++ b/doc/app_overview.md
@@ -29,9 +29,9 @@ Here are the new applications provided by OTBTF.
- **LabelImageSampleSelection**: select patches from a label image
- **DensePolygonClassStatistics**: fast terrain truth polygons statistics
- **TensorflowModelTrain**: training/validation (educational purpose)
-- **TrainClassifierFromDeepFeatures**: train traditionnal classifiers that use
+- **TrainClassifierFromDeepFeatures**: train traditional classifiers that use
features from deep nets (educational/experimental)
-- **ImageClassifierFromDeepFeatures**: use traditionnal classifiers with
+- **ImageClassifierFromDeepFeatures**: use traditional classifiers with
features from deep nets (educational/experimental)
Typically, you could build a pipeline like that without coding a single
diff --git a/otbtf/__init__.py b/otbtf/__init__.py
index 6ee17f9e..70afdf04 100644
--- a/otbtf/__init__.py
+++ b/otbtf/__init__.py
@@ -21,11 +21,14 @@
OTBTF python module
"""
try:
- from otbtf.utils import read_as_np_arr, gdal_open
- from otbtf.dataset import Buffer, PatchesReaderBase, PatchesImagesReader, IteratorBase, RandomIterator, Dataset, \
- DatasetFromPatchesImages
+ from otbtf.utils import read_as_np_arr, gdal_open # noqa
+ from otbtf.dataset import Buffer, PatchesReaderBase, PatchesImagesReader, \
+ IteratorBase, RandomIterator, Dataset, DatasetFromPatchesImages # noqa
except ImportError:
- print("Warning: otbtf.utils and otbtf.dataset were not imported. Using OTBTF without GDAL.")
+ print(
+ "Warning: otbtf.utils and otbtf.dataset were not imported. "
+ "Using OTBTF without GDAL."
+ )
-from otbtf.tfrecords import TFRecords
-from otbtf.model import ModelBase
+from otbtf.tfrecords import TFRecords # noqa
+from otbtf.model import ModelBase # noqa
diff --git a/otbtf/dataset.py b/otbtf/dataset.py
index ae4e7d6a..85b4a4be 100644
--- a/otbtf/dataset.py
+++ b/otbtf/dataset.py
@@ -18,7 +18,8 @@
#
# ==========================================================================*/
"""
-[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/dataset.py){ .md-button }
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/
+tree/master/otbtf/dataset.py){ .md-button }
Contains stuff to help working with TensorFlow and geospatial data in the
OTBTF framework.
@@ -29,15 +30,13 @@ import threading
import time
from abc import ABC, abstractmethod
+from typing import Any, List, Dict, Type, Callable
import numpy as np
import tensorflow as tf
-from typing import Any, List, Dict, Type, Callable
-
-from otbtf.tfrecords import TFRecords
-from otbtf.utils import read_as_np_arr, gdal_open
+import otbtf.tfrecords
+import otbtf.utils
-# --------------------------- Buffer class ------------------------------------
class Buffer:
"""
@@ -78,8 +77,6 @@ class Buffer:
return self.size() == self.max_length
-# ------------------------ PatchesReaderBase class ----------------------------
-
class PatchesReaderBase(ABC):
"""
Base class for patches delivery
@@ -134,8 +131,6 @@ class PatchesReaderBase(ABC):
"""
-# --------------------------------------------- PatchesImagesReader class ----------------------------------------------
-
class PatchesImagesReader(PatchesReaderBase):
"""
This class provides a read access to a set of patches images.
@@ -167,7 +162,7 @@ class PatchesImagesReader(PatchesReaderBase):
):
"""
Params:
- filenames_dict: A dict() structured as follow:
+ filenames_dict: A dict structured as follow:
{
src_name1: [
src1_patches_image_1.tif, ..., src1_patches_image_N.tif
@@ -198,7 +193,7 @@ class PatchesImagesReader(PatchesReaderBase):
# gdal_ds dict
self.gdal_ds = {
- key: [gdal_open(src_fn) for src_fn in src_fns]
+ key: [otbtf.utils.gdal_open(src_fn) for src_fn in src_fns]
for key, src_fns in filenames_dict.items()
}
@@ -227,7 +222,7 @@ class PatchesImagesReader(PatchesReaderBase):
# gdal_ds check
nb_of_patches = {key: 0 for key in self.gdal_ds}
- self.nb_of_channels = dict()
+ self.nb_of_channels = {}
for src_key, ds_list in self.gdal_ds.items():
for gdal_ds in ds_list:
nb_of_patches[src_key] += self._get_nb_of_patches(gdal_ds)
@@ -258,7 +253,7 @@ class PatchesImagesReader(PatchesReaderBase):
if not self.use_streaming:
self.patches_buffer = {
src_key: np.concatenate([
- read_as_np_arr(ds) for ds in src_ds
+ otbtf.utils.read_as_np_arr(ds) for ds in src_ds
], axis=0)
for src_key, src_ds in self.gdal_ds.items()
}
@@ -297,7 +292,7 @@ class PatchesImagesReader(PatchesReaderBase):
index: the sample index. Must be in the [0, self.size) range.
Returns:
- The sample is stored in a dict() with the following structure:
+ The sample is stored in a dict with the following structure:
{
"src_key_0": np.array((psz_y_0, psz_x_0, nb_ch_0)),
"src_key_1": np.array((psz_y_1, psz_x_1, nb_ch_1)),
@@ -403,8 +398,6 @@ class PatchesImagesReader(PatchesReaderBase):
return self.size
-# ----------------------------------------------- IteratorBase class ---------------------------------------------------
-
class IteratorBase(ABC):
"""
Base class for iterators
@@ -415,8 +408,6 @@ class IteratorBase(ABC):
pass
-# ---------------------------- RandomIterator class ---------------------------
-
class RandomIterator(IteratorBase):
"""
Pick a random number in the [0, handler.size) range.
@@ -448,8 +439,6 @@ class RandomIterator(IteratorBase):
np.random.shuffle(self.indices)
-# ------------------------------------------------- Dataset class ------------------------------------------------------
-
class Dataset:
"""
Handles the "mining" of patches.
@@ -468,10 +457,14 @@ class Dataset:
max_nb_of_samples: int = None
):
"""
- :param patches_reader: The patches reader instance
- :param buffer_length: The number of samples that are stored in the buffer
- :param iterator_cls: The iterator class used to generate the sequence of patches indices.
- :param max_nb_of_samples: Optional, max number of samples to consider
+ Params:
+ patches_reader: The patches reader instance
+ buffer_length: The number of samples that are stored in the
+ buffer
+ iterator_cls: The iterator class used to generate the sequence of
+ patches indices.
+ max_nb_of_samples: Optional, max number of samples to consider
+
"""
# patches reader
self.patches_reader = patches_reader
@@ -489,8 +482,8 @@ class Dataset:
self.iterator = iterator_cls(patches_reader=self.patches_reader)
# Get patches sizes and type, of the first sample of the first tile
- self.output_types = dict()
- self.output_shapes = dict()
+ self.output_types = {}
+ self.output_shapes = {}
one_sample = self.patches_reader.get_sample(index=0)
for src_key, np_arr in one_sample.items():
self.output_shapes[src_key] = np_arr.shape
@@ -533,7 +526,7 @@ class Dataset:
drop_remainder: drop remaining samples
"""
- tfrecord = TFRecords(output_dir)
+ tfrecord = otbtf.tfrecords.TFRecords(output_dir)
tfrecord.ds2tfrecord(
self,
n_samples_per_shard=n_samples_per_shard,
@@ -694,8 +687,6 @@ class Dataset:
return self.tot_wait
-# --------------------- DatasetFromPatchesImages class ------------------------
-
class DatasetFromPatchesImages(Dataset):
"""
Handles the "mining" of a set of patches images.
@@ -713,7 +704,7 @@ class DatasetFromPatchesImages(Dataset):
):
"""
Params:
- filenames_dict: A dict() structured as follow:
+ filenames_dict: A dict structured as follow:
{
src_name1: [src1_patches_image1, ..., src1_patches_imageN1],
src_name2: [src2_patches_image2, ..., src2_patches_imageN2],
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/__init__.py b/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
index f84e2e75..d0ec3db4 100644
--- a/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
+++ b/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
@@ -1,5 +1,6 @@
"""
-[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/examples/tensorflow_v2x/deterministic){ .md-button }
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/
+tree/master/otbtf/examples/tensorflow_v2x/deterministic){ .md-button }
This section contains two examples of very simple models that are not
trainable, called **deterministic models**.
@@ -91,4 +92,4 @@ OTB_TF_NSOURCES=2 otbcli_TensorflowModelServe \\
-optim.disabletiling on # Small memory footprint, we can remove tiling
```
-"""
\ No newline at end of file
+"""
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py b/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py
index a2dd7136..b23d86cb 100644
--- a/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py
+++ b/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py
@@ -3,12 +3,15 @@ This code implements a simple model that inputs one multispectral image ("x"),
and computes the euclidean norm of each pixel ("y").
The model is exported as a SavedModel named "l2_norm_savedmodel"
To run the SavedModel:
-otbcli_TensorflowModelServe \
--source1.il image1.tif \
+
+```
+otbcli_TensorflowModelServe \
+-source1.il image1.tif \
-model.dir l2_norm_savedmodel \
--model.fullyconv on \
--out output.tif \
--optim.disabletiling on # Tiling is not helping here, since its a pixel wise op.
+-model.fullyconv on \
+-out output.tif \
+-optim.disabletiling on # Tiling is not helping here (it is a pixel wise op)
+```
"""
import tensorflow as tf
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py b/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py
index 3f41e891..57127c5e 100644
--- a/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py
+++ b/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py
@@ -1,15 +1,20 @@
"""
-This code implements a simple model that inputs two multispectral image ("x1" and "x2"),
+This code implements a simple model that inputs two multispectral image ("x1"
+and "x2"),
and computes the scalar product between each pixels of the two images.
The model is exported as a SavedModel named "scalar_product_savedmodel"
To run the SavedModel:
+
+```
OTB_TF_NSOURCES=2 otbcli_TensorflowModelServe \
--source1.il image1.tif \
--source2.il image2.tif \
--model.dir scalar_product_savedmodel \
--model.fullyconv on \
--out output.tif \
--optim.disabletiling on # Tiling is not helping here, since its a pixel wise op.
+-source1.il image1.tif \
+-source2.il image2.tif \
+-model.dir scalar_product_savedmodel \
+-model.fullyconv on \
+-out output.tif \
+-optim.disabletiling on # (Tiling is not helping here, it is a pixel wise op)
+```
+
"""
import tensorflow as tf
@@ -23,4 +28,3 @@ y = tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
# Create model
model = tf.keras.Model(inputs={"x1": x1, "x2": x2}, outputs={"y": y})
model.save("scalar_product_savedmodel")
-
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
index 605c28aa..2acb0ae9 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
@@ -33,6 +33,10 @@ parser.add_argument(
def create_tfrecords(params):
+ """
+ Create TFRecords.
+
+ """
# Sort patches and labels
patches = sorted(params.xs)
labels = sorted(params.labels)
@@ -58,5 +62,4 @@ def create_tfrecords(params):
if __name__ == "__main__":
- params = parser.parse_args()
- create_tfrecords(params)
+ create_tfrecords(parser.parse_args())
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/helper.py b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
index bcc9036a..868fbb0b 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/helper.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
@@ -39,8 +39,8 @@ def check_files_order(files1, files2):
assert files2
assert len(files1) == len(files2)
- def get_basename(n):
- return "_".join([n.split("_")][:-1])
+ def get_basename(filename):
+ return "_".join([filename.split("_")][:-1])
- for p, l in zip(files1, files2):
- assert get_basename(p) == get_basename(l)
+ for file1, file2 in zip(files1, files2):
+ assert get_basename(file1) == get_basename(file2)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
index 84b8726e..9dab4b9b 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
@@ -1,5 +1,6 @@
"""
-This example shows how to use the otbtf python API to train a deep net from patches-images.
+This example shows how to use the otbtf python API to train a deep net from
+patches-images.
"""
from otbtf import DatasetFromPatchesImages
from otbtf.examples.tensorflow_v2x.fcnn import fcnn_model
@@ -54,7 +55,7 @@ def create_dataset(
xs_filenames: list,
labels_filenames: list,
batch_size: int,
- targets_keys: list = [fcnn_model.TARGET_NAME]
+ targets_keys: list = None
):
"""
Returns a TF dataset generated from an `otbtf.DatasetFromPatchesImages`
@@ -74,7 +75,7 @@ def create_dataset(
# computation is slower than one batch gathering! You can also use a
# custom `Iterator` of your own (default is `RandomIterator`).
# See `otbtf.dataset.Iterator`.
- ds = DatasetFromPatchesImages(
+ dataset = DatasetFromPatchesImages(
filenames_dict={
"input_xs_patches": xs_filenames,
"labels_patches": labels_filenames
@@ -86,16 +87,20 @@ def create_dataset(
# Also, we set the `target_keys` parameter to ask the dataset to deliver
# samples in the form expected by keras, i.e. a tuple of dicts
# (inputs_dict, target_dict).
- tf_ds = ds.get_tf_dataset(
+ tf_ds = dataset.get_tf_dataset(
batch_size=batch_size,
preprocessing_fn=fcnn_model.dataset_preprocessing_fn,
- targets_keys=targets_keys
+ targets_keys=targets_keys or [fcnn_model.TARGET_NAME]
)
return tf_ds
def train(params):
+ """
+ Train from patches images.
+
+ """
# Create TF datasets
ds_train = create_dataset(
params.train_xs, params.train_labels, batch_size=params.batch_size
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index 3fa303c2..caf15a79 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -39,6 +39,10 @@ parser.add_argument(
def train(params):
+ """
+ Train from TFRecords.
+
+ """
# Patches directories must contain 'train' and 'valid' dirs ('test' is not
# required)
train_dir = os.path.join(params.tfrecords_dir, "train")
diff --git a/otbtf/model.py b/otbtf/model.py
index 554782b7..0dca0185 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -18,7 +18,8 @@
#
# ==========================================================================*/
"""
-[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/model.py){ .md-button }
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/
+tree/master/otbtf/model.py){ .md-button }
Base class for models.
"""
@@ -29,6 +30,7 @@ import tensorflow as tf
TensorsDict = Dict[str, tf.keras.engine.keras_tensor.KerasTensor]
+
class ModelBase(abc.ABC):
"""
Base class for all models
@@ -162,7 +164,8 @@ class ModelBase(abc.ABC):
) -> TensorsDict:
"""
Post-process the model outputs.
- Takes the dicts of inputs and outputs, and returns a dict of post-processed outputs.
+ Takes the dicts of inputs and outputs, and returns a dict of
+ post-processed outputs.
The default implementation provides a set of cropped output tensors.
Params:
@@ -234,7 +237,8 @@ class ModelBase(abc.ABC):
def summary(self, strategy=None):
"""
- Wraps the summary printing of the model. When multiworker strategy, only prints if the worker is chief
+ Wraps the summary printing of the model. When multiworker strategy,
+ only prints if the worker is chief
:param strategy: strategy
"""
@@ -253,11 +257,14 @@ class ModelBase(abc.ABC):
show_shapes: annotate with shapes values (True or False)
"""
- assert self.model, "Plot() only works if create_network() has been called beforehand"
+ assert self.model, "Plot() only works if create_network() has been " \
+ "called beforehand"
# When multiworker strategy, only plot if the worker is chief
if not strategy or _is_chief(strategy):
- tf.keras.utils.plot_model(self.model, output_path, show_shapes=show_shapes)
+ tf.keras.utils.plot_model(
+ self.model, output_path, show_shapes=show_shapes
+ )
def _is_chief(strategy):
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 4fe1173c..e5ac0841 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -18,7 +18,8 @@
#
# ==========================================================================*/
"""
-[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/tfrecords.py){ .md-button }
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/
+tree/master/otbtf/tfrecords.py){ .md-button }
The tfrecords module provides an implementation for the TFRecords files
read/write
@@ -29,11 +30,10 @@ import logging
import os
from functools import partial
+from typing import Any, List, Dict, Callable
import tensorflow as tf
from tqdm import tqdm
-import otbtf
-
class TFRecords:
"""
@@ -71,12 +71,13 @@ class TFRecords:
a bytes_list from a string / byte.
"""
if isinstance(value, type(tf.constant(0))):
- value = value.numpy() # BytesList won't unpack a string from an EagerTensor.
+ value = value.numpy() # BytesList won't unpack a string from
+ # an EagerTensor.
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def ds2tfrecord(
self,
- dataset: otbtf.Dataset,
+ dataset: Any,
n_samples_per_shard: int = 100,
drop_remainder: bool = True
):
@@ -93,14 +94,11 @@ class TFRecords:
"""
logging.info("%s samples", dataset.size)
- nb_shards = (dataset.size // n_samples_per_shard)
+ nb_shards = dataset.size // n_samples_per_shard
if not drop_remainder and dataset.size % n_samples_per_shard > 0:
nb_shards += 1
- output_shapes = {
- key: output_shape
- for key, output_shape in dataset.output_shapes.items()
- }
+ output_shapes = dict(dataset.output_shapes.items())
self.save(output_shapes, self.output_shapes_file)
output_types = {
diff --git a/otbtf/utils.py b/otbtf/utils.py
index f2881df5..1c552fbd 100644
--- a/otbtf/utils.py
+++ b/otbtf/utils.py
@@ -18,7 +18,8 @@
#
# ==========================================================================*/
"""
-[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/utils.py){ .md-button }
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/
+tree/master/otbtf/utils.py){ .md-button }
The utils module provides some helpers to read patches using gdal
"""
@@ -26,8 +27,6 @@ from osgeo import gdal
import numpy as np
-# --------------------------------- Helpers -----------------------------------
-
def gdal_open(filename: str):
"""
Open a GDAL raster
--
GitLab
From ada15b7fdc81f5624b1a1acbbe4c69321c51e898 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 29 Mar 2023 21:39:34 +0200
Subject: [PATCH 05/72] FIX: function typing
---
otbtf/dataset.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/otbtf/dataset.py b/otbtf/dataset.py
index 85b4a4be..cf2a0759 100644
--- a/otbtf/dataset.py
+++ b/otbtf/dataset.py
@@ -619,7 +619,7 @@ class Dataset:
drop_remainder: bool = True,
preprocessing_fn: Callable = None,
targets_keys: List[str] = None
- ) -> tf.Dataset:
+ ) -> tf.data.Dataset:
"""
Returns a TF dataset, ready to be used with the provided batch size
--
GitLab
From ea44ff8e156ace84db04d68ae7edfe3a490b7739 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 08:57:05 +0200
Subject: [PATCH 06/72] ADD: otbtf.egg-info, build, site
---
.gitignore | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.gitignore b/.gitignore
index 1ef65aa1..9ec7724f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,6 @@
# Compiled python source #
*.pyc
.idea
+build
+site
+otbtf.egg-info
--
GitLab
From ed5b49791c0b13913018c4704fc709a8f8cbee31 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 08:57:22 +0200
Subject: [PATCH 07/72] FIX: typing in ModelBase
---
otbtf/model.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index 0dca0185..bee1a2e9 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -28,7 +28,7 @@ import abc
import logging
import tensorflow as tf
-TensorsDict = Dict[str, tf.keras.engine.keras_tensor.KerasTensor]
+TensorsDict = Dict[str, Any]
class ModelBase(abc.ABC):
--
GitLab
From 16d43d588162f0b9a9991878a1e5ac816147d9ad Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 09:39:45 +0200
Subject: [PATCH 08/72] CI: build docker image against Ubuntu22.04
---
.gitlab-ci.yml | 28 +++++++++++++++-------------
1 file changed, 15 insertions(+), 13 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b4abcc6c..9846f421 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
variables:
- OTBTF_VERSION: 3.4.0
+ OTBTF_VERSION: 3.5.0
OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory
OTBTF_SRC: /src/otbtf # Local OTBTF source directory
OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory
@@ -18,7 +18,9 @@ variables:
CI_REGISTRY_PUBIMG: $CI_REGISTRY_IMAGE:$OTBTF_VERSION
DOCKERHUB_BASE: mdl4eo/otbtf
DOCKERHUB_IMAGE_BASE: ${DOCKERHUB_BASE}:${OTBTF_VERSION}
-
+ CPU_BASE_IMG: ubuntu:22.04
+ GPU_BASE_IMG: nvidia/cuda:12.1.0-devel-ubuntu22.04
+
workflow:
rules:
- if: $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME =~ /master/ # Execute jobs in merge request context, or commit in master branch
@@ -54,7 +56,7 @@ docker image:
--network="host"
--cache-from $CACHE_IMAGE_BASE
--tag $CACHE_IMAGE_BASE
- --build-arg BASE_IMG="ubuntu:20.04"
+ --build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
"."
- docker push $CACHE_IMAGE_BASE
@@ -68,7 +70,7 @@ docker image:
--build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
- --build-arg BASE_IMG="ubuntu:20.04"
+ --build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
"."
- docker push $CACHE_IMAGE_BUILDER
@@ -83,7 +85,7 @@ docker image:
--build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
- --build-arg BASE_IMG="ubuntu:20.04"
+ --build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
"."
- docker push $BRANCH_IMAGE
@@ -96,13 +98,13 @@ flake8:
extends: .static_analysis_base
script:
- sudo pip install flake8
- - flake8 $OTBTF_SRC/otbtf
+ - flake8 $OTBTF_SRC/otbtf --exclude=tensorflow_v1x
pylint:
extends: .static_analysis_base
script:
- sudo pip install pylint
- - pylint $OTBTF_SRC/otbtf
+ - pylint $OTBTF_SRC/otbtf --ignore=tensorflow_v1x
codespell:
extends: .static_analysis_base
@@ -200,10 +202,10 @@ deploy_cpu:
DOCKERHUB_LATEST: $DOCKERHUB_BASE:latest
script:
# cpu
- - docker build --network='host' --tag $IMAGE_CPU --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS="" .
+ - docker build --network='host' --tag $IMAGE_CPU --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" .
- docker push $IMAGE_CPU
# cpu-dev
- - docker build --network='host' --tag $IMAGE_CPUDEV --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS="" --build-arg KEEP_SRC_OTB=true .
+ - docker build --network='host' --tag $IMAGE_CPUDEV --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" --build-arg KEEP_SRC_OTB=true .
- docker push $IMAGE_CPUDEV
# push images on dockerhub
- echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
@@ -226,16 +228,16 @@ deploy_gpu:
DOCKERHUB_GPUDEV: $DOCKERHUB_IMAGE_BASE-gpu-dev
script:
# gpu-opt
- - docker build --network='host' --tag $IMAGE_GPUOPT --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 .
+ - docker build --network='host' --tag $IMAGE_GPUOPT --build-arg BASE_IMG=$GPU_BASE_IMG .
- docker push $IMAGE_GPUOPT
# gpu-opt-dev
- - docker build --network='host' --tag $IMAGE_GPUOPTDEV --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg KEEP_SRC_OTB=true .
+ - docker build --network='host' --tag $IMAGE_GPUOPTDEV --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
- docker push $IMAGE_GPUOPTDEV
# gpu-basic
- - docker build --network='host' --tag $IMAGE_GPU --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg BZL_CONFIGS="" .
+ - docker build --network='host' --tag $IMAGE_GPU --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg BZL_CONFIGS="" .
- docker push $IMAGE_GPU
# gpu-basic-dev
- - docker build --network='host' --tag $IMAGE_GPUDEV --build-arg BZL_CONFIGS="" --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg KEEP_SRC_OTB=true .
+ - docker build --network='host' --tag $IMAGE_GPUDEV --build-arg BZL_CONFIGS="" --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
- docker push $IMAGE_GPUDEV
# push gpu-basic* images on dockerhub
- echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
--
GitLab
From 30f20b3414fcbe45c675eeeab01186c4dafc0eff Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 09:41:57 +0200
Subject: [PATCH 09/72] COMP: update apt packages
---
Dockerfile | 7 +++++--
tools/docker/build-deps-cli.txt | 1 -
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index ddac9997..0a30693a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -43,7 +43,7 @@ WORKDIR /src/tf
RUN git config --global advice.detachedHead false
### TF
-ARG TF=v2.8.0
+ARG TF=v2.12.0
# Install bazelisk (will read .bazelversion and download the right bazel binary - latest by default)
RUN wget -qO /opt/otbtf/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \
&& chmod +x /opt/otbtf/bin/bazelisk \
@@ -152,8 +152,9 @@ COPY --from=builder /src /src
# System-wide ENV
ENV PATH="/opt/otbtf/bin:$PATH"
ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH"
-ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python:/src/otbtf"
+ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python"
ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
+RUN pip install -e /src/otbtf
# Default user, directory and command (bash is the entrypoint when using 'docker create')
RUN useradd -s /bin/bash -m otbuser
@@ -170,7 +171,9 @@ RUN chown -R otbuser:otbuser /src/otbtf
# This won't prevent ownership problems with volumes if you're not UID 1000
USER otbuser
+
# User-only ENV
+ENV PATH="/home/otbuser/.local/bin:$PATH"
# Test python imports
RUN python -c "import tensorflow"
diff --git a/tools/docker/build-deps-cli.txt b/tools/docker/build-deps-cli.txt
index 67d94187..6b7432f8 100644
--- a/tools/docker/build-deps-cli.txt
+++ b/tools/docker/build-deps-cli.txt
@@ -45,6 +45,5 @@ libsvm-dev
libtinyxml-dev
zlib1g-dev
libgeos++-dev
-libgeos-3.8.0
libgeos-c1v5
libgeos-dev
--
GitLab
From 8b029c6f4e7b6abeb16cd0119d414973779db00a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:24:06 +0200
Subject: [PATCH 10/72] COMP: pip install packaging
---
Dockerfile | 65 +----------------
tools/docker/README.md | 159 -----------------------------------------
2 files changed, 2 insertions(+), 222 deletions(-)
delete mode 100644 tools/docker/README.md
diff --git a/Dockerfile b/Dockerfile
index 0a30693a..5c2d1ae6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,7 +28,7 @@ RUN pip install --no-cache-dir pip --upgrade
# NumPy version is conflicting with system's gdal dep and may require venv
ARG NUMPY_SPEC="==1.22.*"
ARG PROTO_SPEC="==3.20.*"
-RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" \
+RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging \
&& pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
# ----------------------------------------------------------------------------
@@ -55,36 +55,6 @@ ARG BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
# "--compilation_mode opt" is already enabled by default (see tf repo .bazelrc and configure.py)
ARG BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
-# Build
-ARG ZIP_TF_BIN=false
-COPY tools/docker/build-env-tf.sh ./
-RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.git \
- && cd tensorflow \
- && export PATH=$PATH:/opt/otbtf/bin \
- && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/otbtf/lib \
- && bash -c '\
- source ../build-env-tf.sh \
- && ./configure \
- && export TMP=/tmp/bazel \
- && BZL_CMD="build $BZL_TARGETS $BZL_CONFIGS $BZL_OPTIONS" \
- && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" ' \
-# Installation - split here if you want to check files ^
-#RUN cd tensorflow \
- && ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg \
- && pip3 install --no-cache-dir --prefix=/opt/otbtf /tmp/tensorflow_pkg/tensorflow*.whl \
- && ln -s /opt/otbtf/lib/python3.* /opt/otbtf/lib/python3 \
- && cp -P bazel-bin/tensorflow/libtensorflow_cc.so* /opt/otbtf/lib/ \
- && ln -s $(find /opt/otbtf -type d -wholename "*/site-packages/tensorflow/include") /opt/otbtf/include/tf \
- # The only missing header in the wheel
- && cp tensorflow/cc/saved_model/tag_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
- && cp tensorflow/cc/saved_model/signature_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
- # Symlink external libs (required for MKL - libiomp5)
- && for f in $(find -L /opt/otbtf/include/tf -wholename "*/external/*/*.so"); do ln -s $f /opt/otbtf/lib/; done \
- # Compress and save TF binaries
- && ( ! $ZIP_TF_BIN || zip -9 -j --symlinks /opt/otbtf/tf-$TF.zip tensorflow/cc/saved_model/tag_constants.h tensorflow/cc/saved_model/signature_constants.h bazel-bin/tensorflow/libtensorflow_cc.so* /tmp/tensorflow_pkg/tensorflow*.whl ) \
- # Cleaning
- && rm -rf bazel-* /src/tf /root/.cache/ /tmp/*
-
### OTB
ARG GUI=false
ARG OTB=5086d7601d80f2427f4d4d7f2398ec46e7efa300
@@ -113,32 +83,7 @@ RUN apt-get update -y \
&& cmake ../otb/SuperBuild -DCMAKE_INSTALL_PREFIX=/opt/otbtf $OTB_FLAGS \
&& make -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))")
-### OTBTF - copy (without .git/) or clone repository
-COPY . /src/otbtf
-#RUN git clone https://github.com/remicres/otbtf.git /src/otbtf
-RUN ln -s /src/otbtf /src/otb/otb/Modules/Remote/otbtf
-
-# Rebuild OTB with module
-ARG KEEP_SRC_OTB=false
-RUN cd /src/otb/build/OTB/build \
- && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/otbtf/lib \
- && export PATH=$PATH:/opt/otbtf/bin \
- && cmake /src/otb/otb \
- -DCMAKE_INSTALL_PREFIX=/opt/otbtf \
- -DOTB_WRAP_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3 \
- -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \
- -Dtensorflow_include_dir=/opt/otbtf/include/tf \
- # Forcing TF>=2, this Dockerfile hasn't been tested with v1 + missing link for libtensorflow_framework.so in the wheel
- -DTENSORFLOW_CC_LIB=/opt/otbtf/lib/libtensorflow_cc.so.2 \
- -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/lib/python3/site-packages/tensorflow/libtensorflow_framework.so.2 \
- && make install -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") \
- # Cleaning
- && ( $GUI || rm -rf /opt/otbtf/bin/otbgui* ) \
- && ( $KEEP_SRC_OTB || rm -rf /src/otb ) \
- && rm -rf /root/.cache /tmp/*
-
-# Symlink executable python files in PATH
-RUN for f in /src/otbtf/python/*.py; do if [ -x $f ]; then ln -s $f /opt/otbtf/bin/; fi; done
+
# ----------------------------------------------------------------------------
# Final stage
@@ -154,7 +99,6 @@ ENV PATH="/opt/otbtf/bin:$PATH"
ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH"
ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python"
ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
-RUN pip install -e /src/otbtf
# Default user, directory and command (bash is the entrypoint when using 'docker create')
RUN useradd -s /bin/bash -m otbuser
@@ -174,8 +118,3 @@ USER otbuser
# User-only ENV
ENV PATH="/home/otbuser/.local/bin:$PATH"
-
-# Test python imports
-RUN python -c "import tensorflow"
-RUN python -c "import otbtf, tricks"
-RUN python -c "import otbApplication as otb; otb.Registry.CreateApplication('ImageClassifierFromDeepFeatures')"
diff --git a/tools/docker/README.md b/tools/docker/README.md
deleted file mode 100644
index 3dcf38f8..00000000
--- a/tools/docker/README.md
+++ /dev/null
@@ -1,159 +0,0 @@
-# Build with Docker
-Docker build has to be called from the root of the repository (i.e. `docker build .` or `bash tools/docker/multibuild.sh`).
-You can build a custom image using `--build-arg` and several config files :
-- Ubuntu : `BASE_IMG` should accept any version, for additional packages see [build-deps-cli.txt](build-deps-cli.txt) and [build-deps-gui.txt](build-deps-gui.txt).
-- TensorFlow : `TF` arg for the git branch or tag + [build-env-tf.sh](build-env-tf.sh) and BZL_* arguments for the build configuration. `ZIP_TF_BIN` allows you to save compiled binaries if you want to install it elsewhere.
-- OrfeoToolBox : `OTB` arg for the git branch or tag + [build-flags-otb.txt](build-flags-otb.txt) to edit cmake flags. Set `KEEP_SRC_OTB` in order to preserve OTB git directory.
-
-### Base images
-```bash
-UBUNTU=20.04 # or 16.04, 18.04
-CUDA=11.2.2 # or 10.1, 10.2, 11.0.3
-CUDNN=8 # or 7
-IMG=ubuntu:$UBUNTU
-GPU_IMG=nvidia/cuda:$CUDA-cudnn$CUDNN-devel-ubuntu$UBUNTU
-```
-
-### Default arguments
-```bash
-BASE_IMG # mandatory
-CPU_RATIO=1
-GUI=false
-NUMPY_SPEC="==1.19.*"
-TF=v2.8.0
-OTB=8.1.0
-BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package"
-BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
-BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
-ZIP_TF_BIN=false
-KEEP_SRC_OTB=false
-SUDO=true
-
-# NumPy version requirement :
-# TF < 2.4 : "numpy<1.19.0,>=1.16.0"
-# TF >= 2.4 : "numpy==1.19.*"
-# TF >= 2.8 : "numpy==1.22.*"
-```
-
-### Bazel remote cache daemon
-If you just need to rebuild with different GUI or KEEP_SRC arguments, or may be a different branch of OTB, bazel cache will help you to rebuild everything except TF, even if the docker cache was purged (after `docker [system|builder] prune`).
-In order to recycle the cache, bazel config and TF git tag should be exactly the same, any change in [build-env-tf.sh](build-env-tf.sh) and `--build-arg` (if related to bazel env, cuda, mkl, xla...) may result in a fresh new build.
-
-Start a cache daemon - here with max 20GB but 10GB should be enough to save 2 TF builds (GPU and CPU):
-```bash
-mkdir -p $HOME/.cache/bazel-remote
-docker run --detach -u 1000:1000 -v $HOME/.cache/bazel-remote:/data -p 9090:8080 buchgr/bazel-remote-cache --max_size=20
-```
-Then just add ` --network='host'` to the docker build command, or connect bazel to a remote server - see 'BZL_OPTIONS'.
-The other way of docker is a virtual bridge, but you'll need to edit the IP address.
-
-## Images build examples
-```bash
-# Build for CPU using default Dockerfiles args (without AWS, HDFS or GCP support)
-docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:20.04 .
-
-# Clear bazel config var (deactivate default optimizations and unset noaws/nogcp/nohdfs)
-docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS= .
-
-# Enable MKL
-MKL_CONFIG="--config=nogcp --config=noaws --config=nohdfs --config=opt --config=mkl"
-docker build --network='host' -t otbtf:cpu-mkl --build-arg BZL_CONFIGS="$MKL_CONFIG" --build-arg BASE_IMG=ubuntu:20.04 .
-
-# Build for GPU (if you're building for your system only you should edit CUDA_COMPUTE_CAPABILITIES in build-env-tf.sh)
-docker build --network='host' -t otbtf:gpu --build-arg BASE_IMG=nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 .
-
-# Build latest TF and OTB, set git branches/tags to clone
-docker build --network='host' -t otbtf:gpu-dev --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 \
- --build-arg KEEP_SRC_OTB=true --build-arg TF=nightly --build-arg OTB=develop .
-
-# Build old release (TF-2.1)
-docker build --network='host' -t otbtf:oldstable-gpu --build-arg BASE_IMG=nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 \
- --build-arg TF=r2.1 --build-arg NUMPY_SPEC="<1.19" \
- --build-arg BAZEL_OPTIONS="--noincompatible_do_not_split_linking_cmdline --verbose_failures --remote_cache=http://localhost:9090" .
-# You could edit the Dockerfile in order to clone an old branch of the repo instead of copying files from the build context
-```
-
-### Build for another machine and save TF compiled files
-
-Example with TF 2.5
-
-```bash
-# Use same ubuntu and CUDA version than your target machine, beware of CC optimization and CPU compatibility
-# (set env variable CC_OPT_FLAGS and avoid "-march=native" if your Docker's CPU is optimized with AVX2/AVX512 but your target CPU isn't)
-docker build --network='host' -t otbtf:custom --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 \
- --build-arg TF=v2.5.0 --build-arg ZIP_TF_BIN=true .
-# Retrieve zip file
-docker run -v $HOME:/home/otbuser/volume otbtf:custom cp /opt/otbtf/tf-v2.5.0.zip /home/otbuser/volume
-
-# Target machine shell
-cd $HOME
-unzip tf-v2.5.0.zip
-sudo mkdir -p /opt/tensorflow/lib
-sudo mv tf-v2.5.0/libtensorflow_cc* /opt/tensorflow/lib
-# You may need to create a virtualenv, here TF and dependencies are installed next to user's pip packages
-pip3 install -U pip wheel mock six future deprecated "numpy==1.19.*"
-pip3 install --no-deps keras_applications keras_preprocessing
-pip3 install tf-v2.5.0/tensorflow-2.5.0-cp38-cp38-linux_x86_64.whl
-
-TF_WHEEL_DIR="$HOME/.local/lib/python3.8/site-packages/tensorflow"
-# If you installed the wheel as regular user, with root pip it should be in /usr/local/lib/python3.*, or in your virtualenv lib/ directory
-mv tf-v2.5.0/tag_constants.h $TF_WHEEL_DIR/include/tensorflow/cc/saved_model/
-# Then recompile OTB with OTBTF using libraries in /opt/tensorflow/lib and instructions in HOWTOBUILD.md.
-cmake $OTB_GIT \
- -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \
- -DTENSORFLOW_CC_LIB=/opt/tensorflow/lib/libtensorflow_cc.so.2 \
- -Dtensorflow_include_dir=$TF_WHEEL_DIR/include \
- -DTENSORFLOW_FRAMEWORK_LIB=$TF_WHEEL_DIR/libtensorflow_framework.so.2 \
-&& make install -j
-```
-
-### Debug build
-If you fail to build, you can log into the last layer and check CMake logs. Run `docker images`, find the latest layer ID and run a tmp container (`docker run -it d60496d9612e bash`).
-You may also need to split some multi-command layers in the Dockerfile.
-If you see OOM errors during SuperBuild you should decrease CPU_RATIO (e.g. 0.75).
-
-## Container examples
-```bash
-# Pull GPU image and create a new container with your home directory as volume (requires apt package nvidia-docker2 and CUDA>=11.0)
-docker create --gpus=all --volume $HOME:/home/otbuser/volume -it --name otbtf-gpu mdl4eo/otbtf:3.3.2-gpu
-
-# Run interactive
-docker start -i otbtf-gpu
-
-# Run in background
-docker start otbtf-gpu
-docker exec otbtf-gpu python -c 'import tensorflow as tf; print(tf.test.is_gpu_available())'
-```
-
-### Rebuild OTB with more modules
-```bash
-docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf:3.3.2-gpu-dev
-docker start -i otbtf-gpu-dev
-```
-```bash
-# From the container shell:
-sudo -i
-cd /src/otb/otb/Modules/Remote
-git clone https://gitlab.irstea.fr/raffaele.gaetano/otbSelectiveHaralickTextures.git
-cd /src/otb/build/OTB/build
-cmake -DModule_OTBAppSelectiveHaralickTextures=ON /src/otb/otb && make install -j
-```
-
-### Container with GUI
-```bash
-# GUI is disabled by default in order to save space, and because docker xvfb isn't working properly with OpenGL.
-# => otbgui seems OK but monteverdi isn't working
-docker build --network='host' -t otbtf:cpu-gui --build-arg BASE_IMG=ubuntu:20.04 --build-arg GUI=true .
-docker create -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$DISPLAY -it --name otbtf-gui otbtf:cpu-gui
-docker start -i otbtf-gui
-$ mapla
-```
-
-## Common errors
-Build :
-`Error response from daemon: manifest for nvidia/cuda:11.0-cudnn8-devel-ubuntu20.04 not found: manifest unknown: manifest unknown`
-=> Image is missing from dockerhub
-
-Run :
-`failed call to cuInit: UNKNOWN ERROR (303) / no NVIDIA GPU device is present: /dev/nvidia0 does not exist`
-=> Nvidia driver is missing or disabled, make sure to add ` --gpus=all` to your docker run or create command
--
GitLab
From ec2b40b28222aae6a4e2b546f601ff447d605363 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:24:22 +0200
Subject: [PATCH 11/72] DOC: distributed training
---
mkdocs.yml | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/mkdocs.yml b/mkdocs.yml
index be9f396a..9c3b7a0f 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -44,11 +44,14 @@ nav:
- Model generalities: api_model_generalities.md
- Deterministic models: reference/otbtf/examples/tensorflow_v2x/deterministic/__init__.md
- Build and train deep learning models: api_tutorial.md
+ - Distributed training: api_distributed.md
- Python API references:
- dataset: reference/otbtf/dataset.md
- tfrecords: reference/otbtf/tfrecords.md
- model: reference/otbtf/model.md
-- Tensorflow API v1: reference/otbtf/examples/tensorflow_v1x/__init__.md
+- Tensorflow v1:
+ - Tricks (deprecated): deprecated.md
+ - Models examples: reference/otbtf/examples/tensorflow_v1x/__init__.md
# Customization
extra:
--
GitLab
From cc783c90fad8c6f444fb1a0d445db1507eedb072 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:24:40 +0200
Subject: [PATCH 12/72] COMP: version 3.5.0 in setup.py
---
setup.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/setup.py b/setup.py
index 35d216ae..9dd65a95 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setuptools.setup(
name="otbtf",
- version="3.4.0",
+ version="3.5.0",
author="Remi Cresson",
author_email="remi.cresson@inrae.fr",
description="OTBTF: Orfeo ToolBox meets TensorFlow",
@@ -26,5 +26,12 @@ setuptools.setup(
],
packages=setuptools.find_packages(),
python_requires=">=3.6",
- keywords="remote sensing, otb, orfeotoolbox, orfeo toolbox, tensorflow, tf, deep learning, machine learning",
+ keywords=["remote sensing",
+ "otb",
+ "orfeotoolbox",
+ "orfeo toolbox",
+ "tensorflow",
+ "deep learning",
+ "machine learning"
+ ],
)
--
GitLab
From c77cf4a02a8ac4faf46d3ee3804ea8d3d53f230c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:25:20 +0200
Subject: [PATCH 13/72] STYLE: fix pydocstyle
---
otbtf/model.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index bee1a2e9..b3ee7b92 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -240,7 +240,9 @@ class ModelBase(abc.ABC):
Wraps the summary printing of the model. When multiworker strategy,
only prints if the worker is chief
- :param strategy: strategy
+ Params:
+ strategy: strategy
+
"""
if not strategy or _is_chief(strategy):
self.model.summary(line_length=150)
--
GitLab
From a610444de959e58b9e9dbd32c587fd19dd90bfea Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:25:45 +0200
Subject: [PATCH 14/72] DOC: fix typo
---
doc/api_tutorial.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/api_tutorial.md b/doc/api_tutorial.md
index b4787597..443cbc4f 100644
--- a/doc/api_tutorial.md
+++ b/doc/api_tutorial.md
@@ -21,7 +21,7 @@ files
## Datasets
-TensorFlow datasets are the most practical way to feed a network data during
+Tensorflow datasets are the most practical way to feed a network data during
training steps.
In particular, they are very useful to train models with data parallelism using
multiple workers (i.e. multiple GPU devices).
--
GitLab
From 86f62c690e6b3083d5bea2aa52ec13fe2b3e2667 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:26:07 +0200
Subject: [PATCH 15/72] DOC: fix pydocstyle
---
otbtf/examples/tensorflow_v2x/fcnn/helper.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/helper.py b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
index 868fbb0b..cec3d5c1 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/helper.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
@@ -8,7 +8,9 @@ def base_parser():
"""
Create a parser with the base parameters for the training applications
- :return: argparse.ArgumentParser instance
+ Returns:
+ argparse.ArgumentParser instance
+
"""
parser = argparse.ArgumentParser(description="Train a FCNN model")
parser.add_argument(
@@ -32,8 +34,10 @@ def check_files_order(files1, files2):
Except for the last, splits of files1[i] and files2[i] from the "_"
character, must be equal.
- :param files1: list of filenames (str)
- :param files2: list of filenames (str)
+ Params:
+ files1: list of filenames (str)
+ files2: list of filenames (str)
+
"""
assert files1
assert files2
--
GitLab
From ab3b57e7a8ce1a52a38f0080e0fa97cf5cd1a73e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:26:29 +0200
Subject: [PATCH 16/72] ADD: documentation for distributed tranining
---
doc/api_distributed.md | 102 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 102 insertions(+)
create mode 100644 doc/api_distributed.md
diff --git a/doc/api_distributed.md b/doc/api_distributed.md
new file mode 100644
index 00000000..18f29304
--- /dev/null
+++ b/doc/api_distributed.md
@@ -0,0 +1,102 @@
+# Distributed training
+
+Thanks to the new API of Tensorflow since version 2.0, it's very easy to
+perform distributed tranining with the exact same code. Just one single line
+has to be changed!
+
+## Multi-worker mirrored strategy with a SLURM cluster
+
+In the following, we will explain how to use multiple nodes of a GPU cluster
+like the [Jean-Zay supercomputer](http://www.idris.fr/eng/jean-zay/jean-zay-presentation-eng.html),
+using `tf.distribute.MultiWorkerMirroredStrategy`.
+We will adopt the data parallelism scheme, meaning that all the computing
+devices will have replicas of the model, but different chunks of data.
+The principle is that once the forward propagation is performed, the gradients
+from the different devices are aggregated together, and the weights are updated
+on all GPUs.
+
+
+<div align="center" width="50%">
+<img src="http://www.idris.fr/media/images/jean-zay-annonce-01.jpg?id=web%3Ajean-zay%3Ajean-zay-presentation" width=50%>
+<figcaption>Jean-Zay has several hundreds of computing nodes with 4 or 8 GPUs. Copyright Photothèque CNRS/Cyril Frésillon</figcaption>
+</div>
+
+## Code
+
+We can start from the codebase of the fully convolutional model example
+described in the OTBTF [Python API tutorial](#api_tutorial.html).
+
+### Strategy
+
+We change the strategy from `tf.distribute.MirroredStrategy` to
+`tf.distribute.MultiWorkerMirroredStrategy`.
+
+First, we have to instantiate a cluster resolver for SLURM, which is the job
+scheduler of the cluster. The cluster resolver uses the environment variables
+provided by SLURM to grab the useful parameters. On the Jean-Zay computer,
+the port base is **13565**:
+
+```python
+cluster_resolver = tf.distribute.cluster_resolver.SlurmClusterResolver(
+ port_base=13565
+)
+```
+
+Then we specify a communication protocol. The Jean-Zay computer supports
+the NVIDIA NCCL communication protocol, which links tightly GPUs from different
+nodes:
+
+```python
+implementation = tf.distribute.experimental.CommunicationImplementation.NCCL
+communication_options = tf.distribute.experimental.CommunicationOptions(
+ implementation=implementation
+)
+```
+
+Finally, we can replace the strategy with the distributed one:
+
+```python
+#strategy = tf.distribute.MirroredStrategy() # <-- that was before
+strategy = tf.distribute.MultiWorkerMirroredStrategy(
+ cluster_resolver=cluster_resolver,
+ communication_options=communication_options
+)
+```
+
+## SLURM job
+
+Now we have to provide a SLURM job to run our python code over several nodes.
+Below is the content of the *job.slurm* file:
+
+```commandline
+#!/bin/bash
+#SBATCH -A <your_account>@gpu
+#SBATCH --job-name=jobname
+#SBATCH --nodes=4 # number of nodes
+#SBATCH --ntasks-per-node=4 # number of MPI task per node
+#SBATCH --gres=gpu:4 # number of GPU per node
+#SBATCH --cpus-per-task=10 # number of cores per task
+#SBATCH --qos=qos_gpu-t3
+#SBATCH --time=00:59:00
+#SBATCH -C v100-16g # Multiworker strategy wants homogeneous GPUs
+
+cd ${SLURM_SUBMIT_DIR}
+
+# deactivate the HTTP proxy (mandatory for multi-node)
+unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY
+module purge
+module load tensorflow-gpu/py3/2.8.0
+
+export PYTHONPATH=$PYTHONPATH:/path/to/otbtf/
+srun
+python3 /path/to/your_code.py
+```
+
+To submit the job, run the following command:
+```commandline
+sbatch job.slurm
+```
+
+## Reference
+
+- http://www.idris.fr/jean-zay/gpu/jean-zay-gpu-tf-multi.html
--
GitLab
From b5b8edfb273fe01249566509cf06ced2d27e91b9 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:26:43 +0200
Subject: [PATCH 17/72] ADD: documentation for deprecated stuff
---
doc/deprecated.md | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
create mode 100644 doc/deprecated.md
diff --git a/doc/deprecated.md b/doc/deprecated.md
new file mode 100644
index 00000000..c0477962
--- /dev/null
+++ b/doc/deprecated.md
@@ -0,0 +1,38 @@
+!!! Warning
+
+ The `tricks` module is deprecated since OTBTF 2.0
+
+The Tensorflow python API has changed significantly (for the best) after the
+Tensorflow 2.0 release. OTBTF used to provide the `tricks` module, providing
+useful methods to generate the SavedModels, or convert checkpoints into
+SavedModels.
+
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/tricks/){ .md-button }
+
+## What is best in Tensorflow 2.X?
+
+- Shorter and simpler code
+- Easy to build and train a model with Keras, which has become the principal
+interface for Tensorflow
+- More hardware-agnostic than ever: with the exact same code, you can run on
+a single-cpu, GPU, or a pool of GPU servers.
+
+## Major changes between Tensorflow 1 and Tensorflow 2 APIs
+
+Here are a few tips and tricks for people that want to move from
+Tensorflow 1 to Tensorflow 2 API.
+Models built for OTBTF have to take in account the following changes:
+
+- Models built with `otbtf.ModelBase` or `tensorflow.keras.model.Model` have
+no longer to use `tensorflow.compat.v1.placeholder` but
+`tensorflow.keras.Input` instead,
+- Tensorflow variable scopes are no longer used when the training is done from
+Keras,
+- SavedModel can be created directly from the model instance! (as simple as
+`mymodel.save("mymodel_v1_savedmode")`),
+- Switching between single cpu/gpu or multiple computing nodes, distributed
+training, etc. is done using the so-called `tensorflow.Strategy`
+
+!!! Note
+
+ Read our [tutorial](#api_tutorial.html) to know more on working with Keras!
\ No newline at end of file
--
GitLab
From 3c1feccb5ba506da13ea82a14571bed48b20457c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:27:09 +0200
Subject: [PATCH 18/72] DOC: break lines in command-line snippets
---
otbtf/examples/tensorflow_v1x/__init__.py | 305 +++++++++++-----------
1 file changed, 158 insertions(+), 147 deletions(-)
diff --git a/otbtf/examples/tensorflow_v1x/__init__.py b/otbtf/examples/tensorflow_v1x/__init__.py
index 5e885494..c77256a4 100644
--- a/otbtf/examples/tensorflow_v1x/__init__.py
+++ b/otbtf/examples/tensorflow_v1x/__init__.py
@@ -1,7 +1,5 @@
"""
-[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/tree/master/otbtf/examples/tensorflow_v1x/){ .md-button }
-
This section provides some examples of ready-to-use deep learning
architectures built with the TensorFlow API v1 from python.
@@ -9,18 +7,14 @@ architectures built with the TensorFlow API v1 from python.
This section is no longer maintained. We recommend to build and train
models using Keras, which has become the primary interface for Tensorflow
- after version 2.0.
-
- `tricks.py` is here for backward compatibility with codes based on OTBTF 1.x
- and 2.x.
-
-!!! note
+ after version 2.0 (see OTBTF [Python API](#api_tutorial.html)).
- While we recommend to use the Tensorflow API v2, the following examples are
- still great to understand how the models work, in particular in inference
- mode. Models based on the Tensorflow API v1 can be trained easily with the
- `TensorflowModelTrain` application, which allows beginners to focus more
- on the overall process rather than the code.
+ However, while we recommend to use the Tensorflow API v2, the following
+ examples are great to understand how the models work, in particular in
+ inference mode. Models based on the Tensorflow API v1 can be trained easily
+ with the `TensorflowModelTrain` application, which allows beginners to
+ focus more on the overall process rather than the code.
+ Besides, some people still stick with the Tensorflow v1 API: this section
**Table of Contents**
@@ -33,6 +27,8 @@ architectures built with the TensorFlow API v1 from python.
# Simple CNN
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/blob/develop/otbtf/examples/tensorflow_v1x/create_savedmodel_simple_cnn.py){ .md-button }
+
This simple model estimates the class of an input patch of image.
This model consists in successive convolutions/pooling/relu of the input (*x* placeholder).
At some point, the feature map is connected to a dense layer which has N neurons, N being the number of classes we want.
@@ -54,21 +50,28 @@ python create_savedmodel_simple_cnn.py --outdir $modeldir
Note that you can adjust the number of classes for the model with the `--nclasses` option.
+!!! Warning
+
+ If you take a look in *create_savedmode_simple_cnn.py*, you will notice
+ that the `tricks` module is imported at the top of the file.
+ `tricks` is here for backward compatibility with codes based on
+ OTBTF<3.0, and might be deleted in future releases.
+
## Train the model with the CLI
Use **TensorflowModelTrain** in the command line interface to train this model.
```commandLine
-otbcli_TensorflowModelTrain \
--model.dir $modeldir \
--model.saveto "$modeldir/variables/variables" \
--training.source1.il $patches_train -training.source1.placeholder "x" \
--training.source1.patchsizex 1 -training.source1.patchsizey 1 \
--training.source2.il $labels_train -training.source2.placeholder "y" \
--training.source2.patchsizex 1 -training.source2.patchsizey 1 \
--training.targetnodes "optimizer" \
--validation.mode "class" \
--validation.source1.il $patches_valid -validation.source1.name "x" \
+otbcli_TensorflowModelTrain \\
+-model.dir $modeldir \\
+-model.saveto "$modeldir/variables/variables" \\
+-training.source1.il $patches_train -training.source1.placeholder "x" \\
+-training.source1.patchsizex 1 -training.source1.patchsizey 1 \\
+-training.source2.il $labels_train -training.source2.placeholder "y" \\
+-training.source2.patchsizex 1 -training.source2.patchsizey 1 \\
+-training.targetnodes "optimizer" \\
+-validation.mode "class" \\
+-validation.source1.il $patches_valid -validation.source1.name "x" \\
-validation.source2.il $labels_valid -validation.source2.name "prediction"
```
@@ -95,13 +98,13 @@ Since the model is able to estimate the class of the center value of a 16x16
patch, you can run the model over the whole image in patch-based mode.
```
-otbcli_TensorflowModelServe \
--source1.il $image" \
--source1.rfieldx 16 \
--source1.rfieldy 16 \
--source1.placeholder "x" \
--model.dir $modeldir \
--output.names "prediction" \
+otbcli_TensorflowModelServe \\
+-source1.il $image" \\
+-source1.rfieldx 16 \\
+-source1.rfieldy 16 \\
+-source1.placeholder "x" \\
+-model.dir $modeldir \\
+-output.names "prediction" \\
-out $output_classif
```
@@ -121,20 +124,22 @@ If you want to use the model in fully convolutional mode, you have to tell
of the output, 4 in our case.
```commandLine
-otbcli_TensorflowModelServe \
--source1.il $image" \
--source1.rfieldx 16 \
--source1.rfieldy 16 \
--source1.placeholder "x" \
--output.names "prediction" \
--output.spcscale 4 \
--model.dir $modeldir \
--model.fullyconv on \
+otbcli_TensorflowModelServe \\
+-source1.il $image" \\
+-source1.rfieldx 16 \\
+-source1.rfieldy 16 \\
+-source1.placeholder "x" \\
+-output.names "prediction" \\
+-output.spcscale 4 \\
+-model.dir $modeldir \\
+-model.fullyconv on \\
-out $output_classif_fcn
```
# Fully convolutional network
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/blob/develop/otbtf/examples/tensorflow_v1x/create_savedmodel_simple_fcn.py){ .md-button }
+
The `create_savedmodel_simple_fcn.py` script enables you to create a fully
convolutional model which does not use any stride.
@@ -157,26 +162,28 @@ Then you can produce the land cover map at pixel level in fully convolutional
mode:
```commandLine
-otbcli_TensorflowModelServe \
--source1.il $image" \
--source1.rfieldx 16 \
--source1.rfieldy 16 \
--source1.placeholder "x" \
--output.names "prediction" \
--model.dir $modeldir \
--model.fullyconv on \
+otbcli_TensorflowModelServe \\
+-source1.il $image" \\
+-source1.rfieldx 16 \\
+-source1.rfieldy 16 \\
+-source1.placeholder "x" \\
+-output.names "prediction" \\
+-model.dir $modeldir \\
+-model.fullyconv on \\
-out $output_classif
```
# M3Fusion Model
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/blob/develop/otbtf/examples/tensorflow_v1x/create_savedmodel_ienco-m3_patchbased.py){ .md-button }
+
The M3Fusion model (stands for MultiScale/Multimodal/Multitemporal satellite
data fusion) is a model designed to input time series and very high resolution
images.
Benedetti, P., Ienco, D., Gaetano, R., Ose, K., Pensa, R. G., & Dupuy, S.
-(2018). _M3Fusion: A Deep Learning Architecture for Multiscale Multimodal
-Multitemporal Satellite Data Fusion_. IEEE Journal of Selected Topics in
+(2018). *M3Fusion: A Deep Learning Architecture for Multiscale Multimodal
+Multitemporal Satellite Data Fusion*. IEEE Journal of Selected Topics in
Applied Earth Observations and Remote Sensing, 11(12), 4939-4949.
See the original paper [here](https://arxiv.org/pdf/1803.01945).
@@ -217,29 +224,29 @@ that a single big images stack (since "sourceX.il" is an input image list
parameter).
```
-otbcli_TensorflowModelTrain \
--model.dir $modeldir \
--model.saveto "$modeldir/variables/variables" \
--training.source1.il $patches_ts_train \
--training.source1.patchsizex 1 \
--training.source1.patchsizey 1 \
--training.source1.placeholder "x_rnn" \
--training.source2.il $patches_vhr_train \
--training.source2.patchsizex 25 \
--training.source2.patchsizey 25 \
--training.source2.placeholder "x_cnn" \
--training.source3.il $labels_train \
--training.source3.patchsizex 1 \
--training.source3.patchsizey 1 \
--training.source3.placeholder "y" \
--training.targetnodes "optimizer" \
--training.userplaceholders "is_training=true" "drop_rate=0.1" "learning_rate=0.0002" \
--validation.mode "class" -validation.step 1 \
--validation.source1.il $patches_ts_valid \
--validation.source1.name "x_rnn" \
--validation.source2.il $patches_vhr_valid \
--validation.source2.name "x_cnn" \
--validation.source3.il $labels_valid \
+otbcli_TensorflowModelTrain \\
+-model.dir $modeldir \\
+-model.saveto "$modeldir/variables/variables" \\
+-training.source1.il $patches_ts_train \\
+-training.source1.patchsizex 1 \\
+-training.source1.patchsizey 1 \\
+-training.source1.placeholder "x_rnn" \\
+-training.source2.il $patches_vhr_train \\
+-training.source2.patchsizex 25 \\
+-training.source2.patchsizey 25 \\
+-training.source2.placeholder "x_cnn" \\
+-training.source3.il $labels_train \\
+-training.source3.patchsizex 1 \\
+-training.source3.patchsizey 1 \\
+-training.source3.placeholder "y" \\
+-training.targetnodes "optimizer" \\
+-training.userplaceholders "is_training=true" "drop_rate=0.1" "learning_rate=0.0002" \\
+-validation.mode "class" -validation.step 1 \\
+-validation.source1.il $patches_ts_valid \\
+-validation.source1.name "x_rnn" \\
+-validation.source2.il $patches_vhr_valid \\
+-validation.source2.name "x_cnn" \\
+-validation.source3.il $labels_valid \\
-validation.source3.name "prediction"
```
@@ -255,25 +262,27 @@ classes are estimated at the same resolution. This model can be run in
patch-based mode only.
```commandLine
-otbcli_TensorflowModelServe \
--source1.il $ts \
--source1.rfieldx 1 -source1.rfieldy 1 \
--source1.placeholder "x_rnn" \
--source2.il $vhr \
--source2.rfieldx 25 -source2.rfieldy 25 \
--source2.placeholder "x_cnn" \
--model.dir $modeldir \
+otbcli_TensorflowModelServe \\
+-source1.il $ts \\
+-source1.rfieldx 1 -source1.rfieldy 1 \\
+-source1.placeholder "x_rnn" \\
+-source2.il $vhr \\
+-source2.rfieldx 25 -source2.rfieldy 25 \\
+-source2.placeholder "x_cnn" \\
+-model.dir $modeldir \\
-output.names "prediction" -out $output_classif
```
# Maggiori model
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/blob/develop/otbtf/examples/tensorflow_v1x/create_savedmodel_maggiori17_fullyconv.py){ .md-button }
+
This architecture was one of the first to introduce a fully convolutional
model suited for large scale remote sensing images.
Maggiori, E., Tarabalka, Y., Charpiat, G., & Alliez, P. (2016).
-_Convolutional neural networks for large-scale remote-sensing image
-classification_. IEEE Transactions on Geoscience and Remote Sensing, 55(2),
+*Convolutional neural networks for large-scale remote-sensing image
+classification*. IEEE Transactions on Geoscience and Remote Sensing, 55(2),
645-657.
See the original paper [here](https://hal.inria.fr/hal-01350706/document).
@@ -294,21 +303,21 @@ processed with the model, using the `--n_channels` option.
The model perform the semantic segmentation from one single source.
```commandLine
-otbcli_TensorflowModelTrain \
--model.dir $modeldir \
--model.saveto "$modeldir/variables/variables" \
--training.source1.il $patches_image_train \
--training.source1.patchsizex 80 -training.source1.patchsizey 80 \
--training.source1.placeholder "x" \
--training.source2.il $patches_labels_train \
--training.source2.patchsizex 16 -training.source2.patchsizey 16 \
--training.source2.placeholder "y" \
--training.targetnodes "optimizer" \
--training.userplaceholders "is_training=true" "learning_rate=0.0002" \
--validation.mode "class" -validation.step 1 \
--validation.source1.il $patches_image_valid \
--validation.source1.name "x" \
--validation.source2.il $patches_labels_valid \
+otbcli_TensorflowModelTrain \\
+-model.dir $modeldir \\
+-model.saveto "$modeldir/variables/variables" \\
+-training.source1.il $patches_image_train \\
+-training.source1.patchsizex 80 -training.source1.patchsizey 80 \\
+-training.source1.placeholder "x" \\
+-training.source2.il $patches_labels_train \\
+-training.source2.patchsizex 16 -training.source2.patchsizey 16 \\
+-training.source2.placeholder "y" \\
+-training.targetnodes "optimizer" \\
+-training.userplaceholders "is_training=true" "learning_rate=0.0002" \\
+-validation.mode "class" -validation.step 1 \\
+-validation.source1.il $patches_image_valid \\
+-validation.source1.name "x" \\
+-validation.source2.il $patches_labels_valid \\
-validation.source2.name "estimated"
```
@@ -330,26 +339,28 @@ due to the fact that the model keeps only the exact part of the output
features maps.
```
-otbcli_TensorflowModelServe \
--source1.il $image \
--source1.rfieldx 80 -source1.rfieldy 80 \
--source1.placeholder x \
--model.dir $modeldir \
--model.fullyconv on \
--output.names "estimated" \
--output.efieldx 16 -output.efieldy 16 \
+otbcli_TensorflowModelServe \\
+-source1.il $image \\
+-source1.rfieldx 80 -source1.rfieldy 80 \\
+-source1.placeholder x \\
+-model.dir $modeldir \\
+-model.fullyconv on \\
+-output.names "estimated" \\
+-output.efieldx 16 -output.efieldy 16 \\
-out $output_classif
```
# Fully convolutional network with separate channels
+[Source code :fontawesome-brands-github:](https://github.com/remicres/otbtf/blob/develop/otbtf/examples/tensorflow_v1x/create_savedmodel_pxs_fcn.py){ .md-button }
+
It's common that very high resolution products are composed with a
panchromatic channel at high-resolution (Pan), and a multispectral image
generally at lower resolution (MS).
This model inputs separately the two sources (Pan and MS) separately.
-See: Gaetano, R., Ienco, D., Ose, K., & Cresson, R. (2018). A two-branch CNN
-architecture for land cover classification of PAN and MS imagery. Remote
+See: Gaetano, R., Ienco, D., Ose, K., & Cresson, R. (2018). *A two-branch CNN
+architecture for land cover classification of PAN and MS imagery*. Remote
Sensing, 10(11), 1746.
<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_pxs_fcn.png" />
@@ -374,33 +385,33 @@ Here we perform the land cover map at the same resolution as the MS image.
Do do this, we set the MS image as the first source in the **TensorflowModelServe** application.
```
-otbcli_TensorflowModelServe \
--source1.il $ms \
--source1.rfieldx 8 -source1.rfieldy 8 \
--source1.placeholder "x1" \
--source2.il $pan \
--source2.rfieldx 32 -source2.rfieldy 32 \
--source2.placeholder "x2" \
--model.dir $modeldir \
--model.fullyconv on \
--output.names "prediction" \
+otbcli_TensorflowModelServe \\
+-source1.il $ms \\
+-source1.rfieldx 8 -source1.rfieldy 8 \\
+-source1.placeholder "x1" \\
+-source2.il $pan \\
+-source2.rfieldx 32 -source2.rfieldy 32 \\
+-source2.placeholder "x2" \\
+-model.dir $modeldir \\
+-model.fullyconv on \\
+-output.names "prediction" \\
-out $output_classif
```
Note that we could also have set the Pan image as the first source, and tell
the application to use a *spcscale* of 4.
```
-otbcli_TensorflowModelServe \
--source1.il $pan \
--source1.rfieldx 32 -source1.rfieldy 32 \
--source1.placeholder "x2" \
--source2.il $ms \
--source2.rfieldx 8 -source2.rfieldy 8 \
--source2.placeholder "x1" \
--model.dir $modeldir \
--model.fullyconv on \
--output.names "prediction" \
--output.spcscale 4 \
+otbcli_TensorflowModelServe \\
+-source1.il $pan \\
+-source1.rfieldx 32 -source1.rfieldy 32 \\
+-source1.placeholder "x2" \\
+-source2.il $ms \\
+-source2.rfieldx 8 -source2.rfieldy 8 \\
+-source2.placeholder "x1" \\
+-model.dir $modeldir \\
+-model.fullyconv on \\
+-output.names "prediction" \\
+-output.spcscale 4 \\
-out $output_classif
```
@@ -414,15 +425,15 @@ the Pan image resolution.
We hence perform the processing in patch-based mode.
```
-otbcli_TensorflowModelServe \
--source1.il $pan \
--source1.rfieldx 32 -source1.rfieldy 32 \
--source1.placeholder "x2" \
--source2.il $ms \
--source2.rfieldx 8 -source2.rfieldy 8 \
--source2.placeholder "x1" \
--model.dir $modeldir \
--output.names "prediction" \
+otbcli_TensorflowModelServe \\
+-source1.il $pan \\
+-source1.rfieldx 32 -source1.rfieldy 32 \\
+-source1.placeholder "x2" \\
+-source2.il $ms \\
+-source2.rfieldx 8 -source2.rfieldy 8 \\
+-source2.placeholder "x1" \\
+-model.dir $modeldir \\
+-output.names "prediction" \\
-out $output_classif
```
@@ -430,16 +441,16 @@ Note that we could also have set the MS image as the first source, and tell
the application to use a *spcscale* of 0.25.
```
-otbcli_TensorflowModelServe \
--source1.il $ms \
--source1.rfieldx 8 -source1.rfieldy 8 \
--source1.placeholder "x1" \
--source2.il $pan \
--source2.rfieldx 32 -source2.rfieldy 32 \
--source2.placeholder "x2" \
--model.dir $modeldir \
--model.fullyconv on \
--output.names "prediction" \
+otbcli_TensorflowModelServe \\
+-source1.il $ms \\
+-source1.rfieldx 8 -source1.rfieldy 8 \\
+-source1.placeholder "x1" \\
+-source2.il $pan \\
+-source2.rfieldx 32 -source2.rfieldy 32 \\
+-source2.placeholder "x2" \\
+-model.dir $modeldir \\
+-model.fullyconv on \\
+-output.names "prediction" \\
-out $output_classif
```
"""
--
GitLab
From af46df9936ed788ee6b9b033a60673c485c3b368 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 11:59:56 +0200
Subject: [PATCH 19/72] COMP: install otbtf from pip, add user path for python
---
Dockerfile | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 66 insertions(+), 2 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 5c2d1ae6..6a02bd5a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,7 +28,7 @@ RUN pip install --no-cache-dir pip --upgrade
# NumPy version is conflicting with system's gdal dep and may require venv
ARG NUMPY_SPEC="==1.22.*"
ARG PROTO_SPEC="==3.20.*"
-RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging \
+RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" \
&& pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
# ----------------------------------------------------------------------------
@@ -55,6 +55,36 @@ ARG BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
# "--compilation_mode opt" is already enabled by default (see tf repo .bazelrc and configure.py)
ARG BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
+# Build
+ARG ZIP_TF_BIN=false
+COPY tools/docker/build-env-tf.sh ./
+RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.git \
+ && cd tensorflow \
+ && export PATH=$PATH:/opt/otbtf/bin \
+ && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/otbtf/lib \
+ && bash -c '\
+ source ../build-env-tf.sh \
+ && ./configure \
+ && export TMP=/tmp/bazel \
+ && BZL_CMD="build $BZL_TARGETS $BZL_CONFIGS $BZL_OPTIONS" \
+ && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" ' \
+# Installation - split here if you want to check files ^
+#RUN cd tensorflow \
+ && ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg \
+ && pip3 install --no-cache-dir --prefix=/opt/otbtf /tmp/tensorflow_pkg/tensorflow*.whl \
+ && ln -s /opt/otbtf/lib/python3.* /opt/otbtf/lib/python3 \
+ && cp -P bazel-bin/tensorflow/libtensorflow_cc.so* /opt/otbtf/lib/ \
+ && ln -s $(find /opt/otbtf -type d -wholename "*/site-packages/tensorflow/include") /opt/otbtf/include/tf \
+ # The only missing header in the wheel
+ && cp tensorflow/cc/saved_model/tag_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
+ && cp tensorflow/cc/saved_model/signature_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
+ # Symlink external libs (required for MKL - libiomp5)
+ && for f in $(find -L /opt/otbtf/include/tf -wholename "*/external/*/*.so"); do ln -s $f /opt/otbtf/lib/; done \
+ # Compress and save TF binaries
+ && ( ! $ZIP_TF_BIN || zip -9 -j --symlinks /opt/otbtf/tf-$TF.zip tensorflow/cc/saved_model/tag_constants.h tensorflow/cc/saved_model/signature_constants.h bazel-bin/tensorflow/libtensorflow_cc.so* /tmp/tensorflow_pkg/tensorflow*.whl ) \
+ # Cleaning
+ && rm -rf bazel-* /src/tf /root/.cache/ /tmp/*
+
### OTB
ARG GUI=false
ARG OTB=5086d7601d80f2427f4d4d7f2398ec46e7efa300
@@ -83,7 +113,35 @@ RUN apt-get update -y \
&& cmake ../otb/SuperBuild -DCMAKE_INSTALL_PREFIX=/opt/otbtf $OTB_FLAGS \
&& make -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))")
-
+### OTBTF - copy (without .git/) or clone repository
+COPY . /src/otbtf
+#RUN git clone https://github.com/remicres/otbtf.git /src/otbtf
+RUN ln -s /src/otbtf /src/otb/otb/Modules/Remote/otbtf
+
+# Rebuild OTB with module
+ARG KEEP_SRC_OTB=false
+RUN cd /src/otb/build/OTB/build \
+ && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/otbtf/lib \
+ && export PATH=$PATH:/opt/otbtf/bin \
+ && cmake /src/otb/otb \
+ -DCMAKE_INSTALL_PREFIX=/opt/otbtf \
+ -DOTB_WRAP_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3 \
+ -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \
+ -Dtensorflow_include_dir=/opt/otbtf/include/tf \
+ # Forcing TF>=2, this Dockerfile hasn't been tested with v1 + missing link for libtensorflow_framework.so in the wheel
+ -DTENSORFLOW_CC_LIB=/opt/otbtf/lib/libtensorflow_cc.so.2 \
+ -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/lib/python3/site-packages/tensorflow/libtensorflow_framework.so.2 \
+ && make install -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") \
+ # Cleaning
+ && ( $GUI || rm -rf /opt/otbtf/bin/otbgui* ) \
+ && ( $KEEP_SRC_OTB || rm -rf /src/otb ) \
+ && rm -rf /root/.cache /tmp/*
+
+# Symlink executable python files in PATH
+RUN for f in /src/otbtf/python/*.py; do if [ -x $f ]; then ln -s $f /opt/otbtf/bin/; fi; done
+
+# Finally copy all files (python API, doc, etc)
+COPY . /src/otbtf
# ----------------------------------------------------------------------------
# Final stage
@@ -99,6 +157,7 @@ ENV PATH="/opt/otbtf/bin:$PATH"
ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH"
ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python"
ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
+RUN pip install -e /src/otbtf
# Default user, directory and command (bash is the entrypoint when using 'docker create')
RUN useradd -s /bin/bash -m otbuser
@@ -118,3 +177,8 @@ USER otbuser
# User-only ENV
ENV PATH="/home/otbuser/.local/bin:$PATH"
+
+# Test python imports
+RUN python -c "import tensorflow"
+RUN python -c "import otbtf, tricks"
+RUN python -c "import otbApplication as otb; otb.Registry.CreateApplication('ImageClassifierFromDeepFeatures')"
--
GitLab
From 192e48095b6a46947c56e0cdfd0b00158e92658c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 30 Mar 2023 13:44:28 +0200
Subject: [PATCH 20/72] COMP: add packaging module
---
Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 6a02bd5a..33a55b37 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,7 +28,7 @@ RUN pip install --no-cache-dir pip --upgrade
# NumPy version is conflicting with system's gdal dep and may require venv
ARG NUMPY_SPEC="==1.22.*"
ARG PROTO_SPEC="==3.20.*"
-RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" \
+RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging \
&& pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
# ----------------------------------------------------------------------------
--
GitLab
From 4962adb4fc6cf190a45e7f3cd59b57bc1315ada5 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 31 Mar 2023 12:32:02 +0200
Subject: [PATCH 21/72] COMP: TF2.12 and OTB9.0beta
---
Dockerfile | 42 ++++++++++++++++++++++++++----------------
1 file changed, 26 insertions(+), 16 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 33a55b37..69cb62f6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,7 +28,7 @@ RUN pip install --no-cache-dir pip --upgrade
# NumPy version is conflicting with system's gdal dep and may require venv
ARG NUMPY_SPEC="==1.22.*"
ARG PROTO_SPEC="==3.20.*"
-RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging \
+RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging requests \
&& pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
# ----------------------------------------------------------------------------
@@ -37,29 +37,36 @@ FROM otbtf-base AS builder
# A smaller value may be required to avoid OOM errors when building OTB GUI
ARG CPU_RATIO=1
-RUN mkdir -p /src/tf /opt/otbtf/bin /opt/otbtf/include /opt/otbtf/lib
+RUN mkdir -p /src/tf /opt/otbtf/bin /opt/otbtf/include /opt/otbtf/lib/python3
WORKDIR /src/tf
RUN git config --global advice.detachedHead false
### TF
+
ARG TF=v2.12.0
+
# Install bazelisk (will read .bazelversion and download the right bazel binary - latest by default)
RUN wget -qO /opt/otbtf/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \
&& chmod +x /opt/otbtf/bin/bazelisk \
&& ln -s /opt/otbtf/bin/bazelisk /opt/otbtf/bin/bazel
ARG BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package"
-# "--config=opt" will enable 'march=native' (otherwise read comments about CPU compatibility and edit CC_OPT_FLAGS in build-env-tf.sh)
+
+# "--config=opt" will enable 'march=native'
+# (otherwise read comments about CPU compatibility and edit CC_OPT_FLAGS in
+# build-env-tf.sh)
ARG BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
-# "--compilation_mode opt" is already enabled by default (see tf repo .bazelrc and configure.py)
+
+# "--compilation_mode opt" is already enabled by default (see tf repo .bazelrc
+# and configure.py)
ARG BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
# Build
ARG ZIP_TF_BIN=false
COPY tools/docker/build-env-tf.sh ./
-RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.git \
- && cd tensorflow \
+RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.git
+RUN cd tensorflow \
&& export PATH=$PATH:/opt/otbtf/bin \
&& export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/otbtf/lib \
&& bash -c '\
@@ -67,14 +74,15 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
&& ./configure \
&& export TMP=/tmp/bazel \
&& BZL_CMD="build $BZL_TARGETS $BZL_CONFIGS $BZL_OPTIONS" \
- && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" ' \
-# Installation - split here if you want to check files ^
-#RUN cd tensorflow \
+ && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" '
+
+# Installation
+RUN cd tensorflow \
&& ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg \
&& pip3 install --no-cache-dir --prefix=/opt/otbtf /tmp/tensorflow_pkg/tensorflow*.whl \
- && ln -s /opt/otbtf/lib/python3.* /opt/otbtf/lib/python3 \
- && cp -P bazel-bin/tensorflow/libtensorflow_cc.so* /opt/otbtf/lib/ \
- && ln -s $(find /opt/otbtf -type d -wholename "*/site-packages/tensorflow/include") /opt/otbtf/include/tf \
+ && ln -s /opt/otbtf/local/lib/python3.*/* /opt/otbtf/lib/python3 \
+ && ln -s /opt/otbtf/local/bin/* /opt/otbtf/bin \
+ && ln -s $(find /opt/otbtf -type d -wholename "*/dist-packages/tensorflow/include") /opt/otbtf/include/tf \
# The only missing header in the wheel
&& cp tensorflow/cc/saved_model/tag_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
&& cp tensorflow/cc/saved_model/signature_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
@@ -86,6 +94,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
&& rm -rf bazel-* /src/tf /root/.cache/ /tmp/*
### OTB
+
ARG GUI=false
ARG OTB=5086d7601d80f2427f4d4d7f2398ec46e7efa300
ARG OTBTESTS=false
@@ -99,7 +108,7 @@ RUN apt-get update -y \
&& apt-get install --reinstall ca-certificates -y \
&& update-ca-certificates \
&& git clone https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git \
- && cd otb && git checkout $OTB && cd .. \
+ && cd otb && git checkout $OTB && sed -i 's/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g' CMakeLists.txt && cd .. \
&& mkdir -p build \
&& cd build \
&& if $OTBTESTS; then \
@@ -129,8 +138,8 @@ RUN cd /src/otb/build/OTB/build \
-DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \
-Dtensorflow_include_dir=/opt/otbtf/include/tf \
# Forcing TF>=2, this Dockerfile hasn't been tested with v1 + missing link for libtensorflow_framework.so in the wheel
- -DTENSORFLOW_CC_LIB=/opt/otbtf/lib/libtensorflow_cc.so.2 \
- -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/lib/python3/site-packages/tensorflow/libtensorflow_framework.so.2 \
+ -DTENSORFLOW_CC_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_cc.so.2 \
+ -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_framework.so.2 \
&& make install -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") \
# Cleaning
&& ( $GUI || rm -rf /opt/otbtf/bin/otbgui* ) \
@@ -155,7 +164,7 @@ COPY --from=builder /src /src
# System-wide ENV
ENV PATH="/opt/otbtf/bin:$PATH"
ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH"
-ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python"
+ENV PYTHONPATH="/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python"
ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
RUN pip install -e /src/otbtf
@@ -182,3 +191,4 @@ ENV PATH="/home/otbuser/.local/bin:$PATH"
RUN python -c "import tensorflow"
RUN python -c "import otbtf, tricks"
RUN python -c "import otbApplication as otb; otb.Registry.CreateApplication('ImageClassifierFromDeepFeatures')"
+RUN python -c "from osgeo import gdal
\ No newline at end of file
--
GitLab
From 8184127867dd60ff5648ecfece0c83eb41b46e59 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 31 Mar 2023 12:35:16 +0200
Subject: [PATCH 22/72] DOC: update
---
RELEASE_NOTES.txt | 10 ++++++
doc/api_distributed.md | 77 ++++++++++++++++++++++++++++++++----------
doc/app_inference.md | 10 +++---
doc/app_sampling.md | 16 ++++-----
doc/app_training.md | 28 ++++++++-------
doc/docker_build.md | 21 +++++-------
doc/docker_use.md | 30 ++++++++++++----
doc/index.md | 24 ++++++-------
8 files changed, 141 insertions(+), 75 deletions(-)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 490538c2..b96ba1ac 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,3 +1,13 @@
+Version 4.0.0 (3 apr 2023)
+----------------------------------------------------------------
+* Big improvement of the documentation, now have a readthedocs.io for otbtf !
+* Refactoring all the python classes to enforce pep8
+* Ubuntu version: 22.04
+* Cuda version: 12.1.0
+* Tensorflow version: 2.12.0
+* OTB version: 5086d7601d80f2427f4d4d7f2398ec46e7efa300 (pre-9.0beta)
+* Fixed Tensorflow error "Cannot register 2 metrics with the same name"
+
Version 3.4.0 (22 mar 2023)
----------------------------------------------------------------
* Update OTB version to 5086d7601d80f2427f4d4d7f2398ec46e7efa300 (version 8.1.1 with bugfixes on in-memory connection with python bindings)
diff --git a/doc/api_distributed.md b/doc/api_distributed.md
index 18f29304..1478022c 100644
--- a/doc/api_distributed.md
+++ b/doc/api_distributed.md
@@ -4,7 +4,7 @@ Thanks to the new API of Tensorflow since version 2.0, it's very easy to
perform distributed tranining with the exact same code. Just one single line
has to be changed!
-## Multi-worker mirrored strategy with a SLURM cluster
+## Overview
In the following, we will explain how to use multiple nodes of a GPU cluster
like the [Jean-Zay supercomputer](http://www.idris.fr/eng/jean-zay/jean-zay-presentation-eng.html),
@@ -21,11 +21,42 @@ on all GPUs.
<figcaption>Jean-Zay has several hundreds of computing nodes with 4 or 8 GPUs. Copyright Photothèque CNRS/Cyril Frésillon</figcaption>
</div>
-## Code
+## Python code
We can start from the codebase of the fully convolutional model example
described in the OTBTF [Python API tutorial](#api_tutorial.html).
+### Dataset
+
+For distributed training, we recommend to use the TFRecords rather than the
+Patch based images.
+This has two advantages:
+
+- Performance in terms of I/O
+- `otbtf` can be imported without anything else than `tensorflow` as
+dependency. Indeed, the `otbtf.TFRecords` class just needs the `tensorflow`
+module to work.
+
+!!! Info
+
+ When imported, OTBTF tries to import the GDAL-related classes (e.g.
+ `PatchesImagesReader`) and skip the import if an `ImportError` occurs (i.e.
+ when GDAL is not present in the environment). This allows to safely use the
+ other classes that rely purely on the `tensorflow` module (e.g.
+ `otbtf.ModelBase`, `otbtf.TFRecords`, etc.).
+
+### Prerequisites
+
+To use OTBTF on environment where only Tensorflow is available, you can just
+clone the OTBTF repository somewhere and install it in your favorite virtual
+environment with `pip`. Or you can also just update the `PYTHONPATH` to include
+the *otbtf* folder. You just have to be able to perform the import of the
+module from python code:
+
+```python
+import otbtf
+```
+
### Strategy
We change the strategy from `tf.distribute.MirroredStrategy` to
@@ -63,29 +94,38 @@ strategy = tf.distribute.MultiWorkerMirroredStrategy(
)
```
+The rest of the code is identical.
+
+!!! Warning
+
+ Be careful when calling `mymodel.save()` to export the SavedModel. When
+ multiple nodes are used in parallel, this can lead to a corrupt save.
+ One good practice is to defer the call only to the master worker (e.g. node
+ 0). You can identify the master worker using `otbtf.model._is_chief()`.
+
## SLURM job
Now we have to provide a SLURM job to run our python code over several nodes.
Below is the content of the *job.slurm* file:
```commandline
-#!/bin/bash
-#SBATCH -A <your_account>@gpu
-#SBATCH --job-name=jobname
-#SBATCH --nodes=4 # number of nodes
-#SBATCH --ntasks-per-node=4 # number of MPI task per node
-#SBATCH --gres=gpu:4 # number of GPU per node
+#!/bin/bash
+#SBATCH -A <your_account>@gpu
+#SBATCH --job-name=<job_name>
+#SBATCH --nodes=4 # number of nodes
+#SBATCH --ntasks-per-node=4 # number of MPI task per node
+#SBATCH --gres=gpu:4 # number of GPU per node
#SBATCH --cpus-per-task=10 # number of cores per task
-#SBATCH --qos=qos_gpu-t3
+#SBATCH --qos=qos_gpu-t3
#SBATCH --time=00:59:00
#SBATCH -C v100-16g # Multiworker strategy wants homogeneous GPUs
-
-cd ${SLURM_SUBMIT_DIR}
-
-# deactivate the HTTP proxy (mandatory for multi-node)
-unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY
-module purge
-module load tensorflow-gpu/py3/2.8.0
+
+cd ${SLURM_SUBMIT_DIR}
+
+# deactivate the HTTP proxy (mandatory for multi-node)
+unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY
+module purge
+module load tensorflow-gpu/py3/2.8.0
export PYTHONPATH=$PYTHONPATH:/path/to/otbtf/
srun
@@ -97,6 +137,7 @@ To submit the job, run the following command:
sbatch job.slurm
```
-## Reference
+## References
-- http://www.idris.fr/jean-zay/gpu/jean-zay-gpu-tf-multi.html
+- [Jean-Zay users documentation](https://jean-zay-doc.readthedocs.io/en/latest/)
+- [Official Jean-Zay documentation](http://www.idris.fr/jean-zay/gpu/jean-zay-gpu-tf-multi.html)
diff --git a/doc/app_inference.md b/doc/app_inference.md
index 1e369e03..d50a7c03 100644
--- a/doc/app_inference.md
+++ b/doc/app_inference.md
@@ -39,7 +39,7 @@ will create.
## TensorflowModelServe
-The **TensorflowModelServe** application performs the inference, it can be used
+The `TensorflowModelServe` application performs the inference, it can be used
to produce an output raster with the specified tensors.
Thanks to the streaming mechanism, very large images can be produced.
The application uses the `TensorflowModelFilter` and a `StreamingFilter` to
@@ -95,9 +95,9 @@ otbcli_TensorflowModelServe --help
To use classic classifiers performing on a deep learning model features, one
can use a traditional classifier generated from the
-**TrainClassifierFromDeepFeatures** application, in the
-**ImageClassifierFromDeepFeatures** application, which implements the same
-approach with the official OTB **ImageClassifier** application.
+`TrainClassifierFromDeepFeatures` application, in the
+`ImageClassifierFromDeepFeatures` application, which implements the same
+approach with the official OTB `ImageClassifier` application.
The application description can be displayed using:
@@ -118,7 +118,7 @@ the end of the training step.
After this step, we use the trained model to produce the entire map of forest
over the whole Spot-7 image.
-For this, we use the **TensorflowModelServe** application to produce the *
+For this, we use the `TensorflowModelServe` application to produce the *
*prediction** tensor output for the entire image.
```commandLine
diff --git a/doc/app_sampling.md b/doc/app_sampling.md
index 0d09037a..b79ba7f4 100644
--- a/doc/app_sampling.md
+++ b/doc/app_sampling.md
@@ -95,9 +95,9 @@ otbcli_PatchesExtraction --help
## DensePolygonClassStatistics
-This application is a clone of the [PolygonClassStatistics](https://www.orfeo-toolbox.org/CookBook/Applications/app_PolygonClassStatistics.html)
-modified to use rasterization instead of vector based approach, making it
-faster.
+This application is a clone of the [`PolygonClassStatistics`](https://www.orfeo-toolbox.org/CookBook/Applications/app_PolygonClassStatistics.html)
+application from OTB modified to use rasterization instead of vector based
+approach, making it faster.
The application description can be displayed using:
@@ -108,7 +108,7 @@ otbcli_DensePolygonClassStatistics --help
## LabelImageSampleSelection
This application extracts points from an input label image. This application
-is like "SampleSelection", but uses an input label image, rather than an input
+is like `SampleSelection`, but uses an input label image, rather than an input
vector data. It produces a vector data containing a set of points centered on
the pixels of the input label image. The user can control the number of
points. The default strategy consists in producing the same number of points
@@ -131,24 +131,24 @@ forest / non-forest polygons.
First, we compute statistics of the vector data : how many points can we sample
inside objects, and how many objects in each class.
-We use the **PolygonClassStatistics** application of OTB.
+We use the `PolygonClassStatistics` application of OTB.
```commandline
otbcli_PolygonClassStatistics -vec terrain_truth.shp -field class \
-in spot7.tif -out vec_stats.xml
```
-Then, we will select some samples with the **SampleSelection** application of
+Then, we will select some samples with the `SampleSelection` application of
the existing machine learning framework of OTB.
Since the terrain truth is sparse, we want to sample randomly points in
-polygons with the default strategy of the **SampleSelection** OTB application.
+polygons with the default strategy of the `SampleSelection` OTB application.
```
otbcli_SampleSelection -in spot7.tif -vec terrain_truth.shp \
-instats vec_stats.xml -field class -out points.shp
```
-Now we extract the patches with the **PatchesExtraction** application.
+Now we extract the patches with the `PatchesExtraction` application.
We want to produce one image of 16x16 patches, and one image for the
corresponding labels.
diff --git a/doc/app_training.md b/doc/app_training.md
index 30dacb10..5ea6efaa 100644
--- a/doc/app_training.md
+++ b/doc/app_training.md
@@ -3,14 +3,16 @@
This section is for educational purposes. No coding skills are required,
and it's easy to train an existing model built with the Tensorflow API
v1. To have a full control over the model implementation and training
- process, the Tensorflow API v2 with Keras is the way to go.
+ process, the Tensorflow API v2 with Keras is the way to go.
+ If you are interested in more similar examples, please read the
+ [Tensorflow v1 models examples](reference/otbtf/examples/tensorflow_v1x/__init__.md)
## TensorflowModelTrain
-Here we assume that you have produced patches using the **PatchesExtraction**
+Here we assume that you have produced patches using the `PatchesExtraction`
application, and that you have a **SavedModel** stored in a directory somewhere
on your filesystem.
-The **TensorflowModelTrain** application performs the training, validation (
+The `TensorflowModelTrain` application performs the training, validation (
against test dataset, and against validation dataset) providing the usual
metrics that machine learning frameworks provide (confusion matrix, recall,
precision, f-score, ...).
@@ -33,15 +35,15 @@ _User placeholders_ can be set separately for
training (`training.userplaceholders`) and
validation (`validation.userplaceholders`).
The `validation.userplaceholders` can be useful if you have a model that
-behaves differently depending the given placeholder.
+behaves differently depending on the given placeholder.
Let's take the example of dropout: it's nice for training, but you have to
disable it to use the model at inference time.
Hence you will pass a placeholder with "dropout\_rate=0.3" for training and "
dropout\_rate=0.0" for validation.
Of course, one can train models from handmade python code: to import the
patches images, a convenient method consist in reading patches images as numpy
-arrays using OTB applications (e.g. **ExtractROI**) or GDAL, then do a
-np.reshape to the dimensions wanted.
+arrays using OTB applications (e.g. `ExtractROI`) or GDAL, then do a
+`numpy.reshape` to the dimensions wanted.
@@ -62,9 +64,9 @@ features?
This is possible thank to two new applications that uses the existing
training/classification applications of OTB:
-**TrainClassifierFromDeepFeatures**: is a composite application that wire the *
-*TensorflowModelServe** application output into the existing official *
-*TrainImagesClassifier** application.
+`TrainClassifierFromDeepFeatures` is a composite application that wire
+`TensorflowModelServe` application output into the existing official
+`TrainImagesClassifier` application.
The application description can be displayed using:
@@ -79,9 +81,9 @@ We assume that we have already followed the
the end of the patches extraction.
Now we have two images for patches and labels.
-We can split them to distinguish test/validation groups (with the **ExtractROI**
+We can split them to distinguish test/validation groups (with the `ExtractROI`
application for instance).
-But here, we will just perform some fine tuning of our model.
+But here, we will just perform some fine-tuning of our model.
The **SavedModel** is located in the `outmodel` directory.
Our model is quite basic: it has two input placeholders, **x1** and **y1**
respectively for input patches (with size 16x16) and input reference labels (
@@ -89,10 +91,10 @@ with size 1x1).
We named **prediction** the tensor that predict the labels and the optimizer
that perform the stochastic gradient descent is an operator named **optimizer
**.
-We perform the fine tuning and we export the new model variables directly in
+We perform the fine-tuning and we export the new model variables directly in
the _outmodel/variables_ folder, overwriting the existing variables of the
model.
-We use the **TensorflowModelTrain** application to perform the training of this
+We use the `TensorflowModelTrain` application to perform the training of this
existing model.
```
diff --git a/doc/docker_build.md b/doc/docker_build.md
index b2a32ec0..c7a3c1e9 100644
--- a/doc/docker_build.md
+++ b/doc/docker_build.md
@@ -16,11 +16,8 @@ order to preserve OTB git directory.
### Base images
```bash
-UBUNTU=20.04 # or 16.04, 18.04
-CUDA=11.2.2 # or 10.1, 10.2, 11.0.3
-CUDNN=8 # or 7
-IMG=ubuntu:$UBUNTU
-GPU_IMG=nvidia/cuda:$CUDA-cudnn$CUDNN-devel-ubuntu$UBUNTU
+CPU_IMG=ubuntu:22.04
+GPU_IMG=nvidia/cuda:12.1.0-devel-ubuntu22.04
```
### Default arguments
@@ -30,7 +27,7 @@ BASE_IMG # mandatory
CPU_RATIO=1
GUI=false
NUMPY_SPEC="==1.19.*"
-TF=v2.8.0
+TF=v2.12.0
OTB=8.1.0
BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package"
BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
@@ -74,28 +71,28 @@ address.
```bash
# Build for CPU using default Dockerfiles args (without AWS, HDFS or GCP
# support)
-docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:20.04 .
+docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:22.04 .
# Clear bazel config var (deactivate default optimizations and unset
# noaws/nogcp/nohdfs)
docker build --network='host' -t otbtf:cpu \
- --build-arg BASE_IMG=ubuntu:20.04 \
+ --build-arg BASE_IMG=ubuntu:22.04 \
--build-arg BZL_CONFIGS= .
# Enable MKL
MKL_CONFIG="--config=nogcp --config=noaws --config=nohdfs --config=opt --config=mkl"
docker build --network='host' -t otbtf:cpu-mkl \
--build-arg BZL_CONFIGS="$MKL_CONFIG" \
- --build-arg BASE_IMG=ubuntu:20.04 .
+ --build-arg BASE_IMG=ubuntu:22.04 .
# Build for GPU (if you're building for your system only you should edit
# CUDA_COMPUTE_CAPABILITIES in build-env-tf.sh)
docker build --network='host' -t otbtf:gpu \
- --build-arg BASE_IMG=nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 .
+ --build-arg BASE_IMG=nvidia/cuda:12.1.0-devel-ubuntu22.04 .
# Build latest TF and OTB, set git branches/tags to clone
docker build --network='host' -t otbtf:gpu-dev \
- --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 \
+ --build-arg BASE_IMG=nvidia/cuda:12.1.0-devel-ubuntu22.04 \
--build-arg KEEP_SRC_OTB=true \
--build-arg TF=nightly \
--build-arg OTB=develop .
@@ -205,7 +202,7 @@ OTB GUI seems OK but monteverdi isn't working
```bash
docker build --network='host' -t otbtf:cpu-gui \
- --build-arg BASE_IMG=ubuntu:20.04 \
+ --build-arg BASE_IMG=ubuntu:22.04 \
--build-arg GUI=true .
docker create -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$DISPLAY -it \
--name otbtf-gui otbtf:cpu-gui
diff --git a/doc/docker_use.md b/doc/docker_use.md
index 78d6baa9..01e75bf0 100644
--- a/doc/docker_use.md
+++ b/doc/docker_use.md
@@ -1,10 +1,26 @@
# Install from docker
+We recommend to use OTBTF from official docker images.
+
+Latest CPU-only docker image:
+
+```commandline
+docker pull mdl4eo/otbtf:3.4.0-cpu
+```
+
+Latest GPU-ready docker image:
+
+```commandline
+docker pull mdl4eo/otbtf:3.4.0-gpu
+```
+
+Read more in the following sections.
+
## Latest images
Here is the list of the latest OTBTF docker images hosted on
[dockerhub](https://hub.docker.com/u/mdl4eo).
-Since OTBTF >= 3.2.1 you can find latest docker images on
+Since OTBTF >= 3.2.1 you can find the latest docker images on
[gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry).
| Name | Os | TF | OTB | Description | Dev files | Compute capability |
@@ -25,12 +41,6 @@ The list of older releases is available [here](#older-images).
Since r2.4, development-ready images have the source in `/src/` and are
tagged "...-dev".
-## Build your own images
-
-If you want to use optimization flags, change GPUs compute capability, etc.
-you can build your own docker image using the provided dockerfile.
-See the [docker build documentation](../tools/docker/README.md).
-
## GPU enabled docker
In Linux, this is quite straightforward.
@@ -88,6 +98,12 @@ Troubleshooting:
https://docs.nvidia.com/cuda/wsl-user-guide/index.html
https://docs.docker.com/docker-for-windows/wsl/#gpu-support
+## Build your own images
+
+If you want to use optimization flags, change GPUs compute capability, etc.
+you can build your own docker image using the provided dockerfile.
+See the [docker build documentation](#docker_build.html).
+
## Older images
Here you can find the list of older releases of OTBTF:
diff --git a/doc/index.md b/doc/index.md
index 3f30a209..4bb28c76 100644
--- a/doc/index.md
+++ b/doc/index.md
@@ -16,7 +16,7 @@
This remote module of the [Orfeo ToolBox](https://www.orfeo-toolbox.org)
-provides a generic, multi purpose deep learning framework, targeting remote
+provides a generic, multi-purpose deep learning framework, targeting remote
sensing images processing. It contains a set of new process objects for OTB
that internally invoke [Tensorflow](https://www.tensorflow.org/), and new [OTB
applications](#otb-applications) to perform deep learning with real-world
@@ -56,6 +56,7 @@ files, which is quite suited for distributed training. Read more in the
Below are some screen captures of deep learning applications performed at
large scale with OTBTF.
+
- Landcover mapping (Spot-7 images --> Building map using semantic
segmentation)
@@ -97,16 +98,15 @@ Every one can **contribute** to OTBTF. Just open a PR :)
## Additional resources
+- The [*test* folder](https://github.com/remicres/otbtf/tree/master/test/)
+of this repository contains various use-cases with commands, python codes, and
+input/baseline data,
+- This [book](https://doi.org/10.1201/9781003020851) contains 130 pages to
+learn how to use OTBTF with OTB and QGIS to perform various kind of deep
+learning sorcery on remote sensing images (patch-based classification for
+landcover mapping, semantic segmentation of buildings, optical image
+restoration from joint SAR/Optical time series): *Cresson, R. (2020). Deep
+Learning for Remote Sensing Images with Open Source Software. CRC Press.*
- A small [tutorial](https://mdl4eo.irstea.fr/2019/01/04/an-introduction-to-deep-learning-on-remote-sensing-images-tutorial/) on MDL4EO's blog
-- in the `python` folder are provided some [ready-to-use deep networks, with
-documentation and scientific references](doc/EXAMPLES.md).
-- A [book](https://doi.org/10.1201/9781003020851): *Cresson, R. (2020).
-Deep Learning for Remote Sensing Images with Open Source Software. CRC Press.*
-Use QGIS, OTB and Tensorflow to perform various kind of deep learning sorcery
-on remote sensing images (patch-based classification for landcover mapping,
-semantic segmentation of buildings, optical image restoration from joint
-SAR/Optical time series).
- Check [our repository](https://github.com/remicres/otbtf_tutorials_resources)
-containing stuff (data and models) to begin with with!
-- Finally, take a look in the `test` folder. You will find plenty of command
-lines for applications tests!
+containing stuff (data and models) to begin with!
--
GitLab
From 1687a8cc0cae69921fbd8da4b6241ba78b1d031a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 31 Mar 2023 18:12:41 +0200
Subject: [PATCH 23/72] FIX: missing quote
---
Dockerfile | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 69cb62f6..280eb639 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -149,9 +149,6 @@ RUN cd /src/otb/build/OTB/build \
# Symlink executable python files in PATH
RUN for f in /src/otbtf/python/*.py; do if [ -x $f ]; then ln -s $f /opt/otbtf/bin/; fi; done
-# Finally copy all files (python API, doc, etc)
-COPY . /src/otbtf
-
# ----------------------------------------------------------------------------
# Final stage
FROM otbtf-base
@@ -191,4 +188,4 @@ ENV PATH="/home/otbuser/.local/bin:$PATH"
RUN python -c "import tensorflow"
RUN python -c "import otbtf, tricks"
RUN python -c "import otbApplication as otb; otb.Registry.CreateApplication('ImageClassifierFromDeepFeatures')"
-RUN python -c "from osgeo import gdal
\ No newline at end of file
+RUN python -c "from osgeo import gdal"
\ No newline at end of file
--
GitLab
From 2d236e9fc77313b8bce43092a72b2052cb9d36e9 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 31 Mar 2023 22:49:22 +0200
Subject: [PATCH 24/72] CI: docker in shell
---
.gitlab-ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9846f421..f2246b10 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,9 +36,9 @@ stages:
.docker_build_base:
allow_failure: false
tags: [godzilla]
- image: docker:latest
- services:
- - name: docker:dind
+# image: docker:latest
+# services:
+# - name: docker:dind
before_script:
- echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
timeout: 10 hours
--
GitLab
From 3e1d73d22867b9590e23366d7f84fdd978a233c4 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 12:06:32 +0200
Subject: [PATCH 25/72] WIP: debug compile tests
---
.gitlab-ci.yml | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f2246b10..20d4ea74 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -67,7 +67,7 @@ docker image:
--cache-from $CACHE_IMAGE_BASE
--cache-from $CACHE_IMAGE_BUILDER
--tag $CACHE_IMAGE_BUILDER
- --build-arg OTBTESTS="true"
+# --build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
@@ -82,7 +82,7 @@ docker image:
--cache-from $BRANCH_IMAGE
--cache-from $DEV_IMAGE
--tag $BRANCH_IMAGE
- --build-arg OTBTESTS="true"
+# --build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
@@ -130,7 +130,8 @@ ctest:
extends: .tests_base
stage: Test
script:
- - cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow # Run ctest
+# - cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow # Run ctest
+ - echo "todo: delete this"
after_script:
- cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
--
GitLab
From acdcc395545ad72f37b95c2ea0522eff209b9514 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 12:07:11 +0200
Subject: [PATCH 26/72] WIP: debug compile tests
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 20d4ea74..fa257dd8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -67,12 +67,12 @@ docker image:
--cache-from $CACHE_IMAGE_BASE
--cache-from $CACHE_IMAGE_BUILDER
--tag $CACHE_IMAGE_BUILDER
-# --build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
"."
+# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
- >
docker build
@@ -82,12 +82,12 @@ docker image:
--cache-from $BRANCH_IMAGE
--cache-from $DEV_IMAGE
--tag $BRANCH_IMAGE
-# --build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
"."
+# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
.static_analysis_base:
--
GitLab
From a55d69b59a646285efc78db536e4946d217c76be Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 12:09:32 +0200
Subject: [PATCH 27/72] WIP: debug compile tests
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fa257dd8..fc3dff0a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -130,8 +130,8 @@ ctest:
extends: .tests_base
stage: Test
script:
-# - cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow # Run ctest
- echo "todo: delete this"
+# - cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow # Run ctest
after_script:
- cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
--
GitLab
From 9e42d2dec853559abddb3253a11d8bdec5b9cba0 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 12:10:13 +0200
Subject: [PATCH 28/72] WIP: debug compile tests
---
.gitlab-ci.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fc3dff0a..a3fc5f1a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -130,8 +130,7 @@ ctest:
extends: .tests_base
stage: Test
script:
- - echo "todo: delete this"
-# - cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow # Run ctest
+ - echo "todo: delete this" # cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow # Run ctest
after_script:
- cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
--
GitLab
From af15c601c3ffc38b41fa1eef1f8cd8fdccb5564c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 12:11:31 +0200
Subject: [PATCH 29/72] WIP: debug compile tests
---
.gitlab-ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a3fc5f1a..96e9b7f9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -67,12 +67,12 @@ docker image:
--cache-from $CACHE_IMAGE_BASE
--cache-from $CACHE_IMAGE_BUILDER
--tag $CACHE_IMAGE_BUILDER
+# --build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
"."
-# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
- >
docker build
@@ -82,12 +82,12 @@ docker image:
--cache-from $BRANCH_IMAGE
--cache-from $DEV_IMAGE
--tag $BRANCH_IMAGE
+# --build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
"."
-# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
.static_analysis_base:
@@ -130,7 +130,7 @@ ctest:
extends: .tests_base
stage: Test
script:
- - echo "todo: delete this" # cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow # Run ctest
+ - cd $OTB_BUILD/ #&& sudo ctest -L OTBTensorflow # Run ctest
after_script:
- cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
--
GitLab
From 1ba007ef05ff1070dca9ad1a502669121dfd8ab8 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 12:11:57 +0200
Subject: [PATCH 30/72] WIP: debug compile tests
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 96e9b7f9..460dd703 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -67,12 +67,12 @@ docker image:
--cache-from $CACHE_IMAGE_BASE
--cache-from $CACHE_IMAGE_BUILDER
--tag $CACHE_IMAGE_BUILDER
-# --build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
"."
+# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
- >
docker build
@@ -82,12 +82,12 @@ docker image:
--cache-from $BRANCH_IMAGE
--cache-from $DEV_IMAGE
--tag $BRANCH_IMAGE
-# --build-arg OTBTESTS="true"
--build-arg KEEP_SRC_OTB="true"
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
"."
+# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
.static_analysis_base:
--
GitLab
From 827eca06702d6d7916f9acaf2fb294de44ec8e58 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 14:39:55 +0200
Subject: [PATCH 31/72] WIP: debug compile tests
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 460dd703..61e96b7e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -131,8 +131,8 @@ ctest:
stage: Test
script:
- cd $OTB_BUILD/ #&& sudo ctest -L OTBTensorflow # Run ctest
- after_script:
- - cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
+# after_script:
+# - cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
.applications_test_base:
extends: .tests_base
--
GitLab
From 37359cd160c58c5f641e6927cadc125fcd904d00 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 14:50:32 +0200
Subject: [PATCH 32/72] WIP: debug compile tests
---
.gitlab-ci.yml | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 61e96b7e..47e5867a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,9 +36,9 @@ stages:
.docker_build_base:
allow_failure: false
tags: [godzilla]
-# image: docker:latest
-# services:
-# - name: docker:dind
+ image: docker:latest
+ services:
+ - name: docker:dind
before_script:
- echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
timeout: 10 hours
@@ -131,8 +131,9 @@ ctest:
stage: Test
script:
- cd $OTB_BUILD/ #&& sudo ctest -L OTBTensorflow # Run ctest
-# after_script:
-# - cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
+ - touch $OTB_TEST_DIR/toto.txt
+ after_script:
+ - cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
.applications_test_base:
extends: .tests_base
--
GitLab
From 5064441f6dcfff44c3b5fd56f7c354212d8385e4 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 17:30:52 +0200
Subject: [PATCH 33/72] WIP: debug compile tests
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 47e5867a..98ef447b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -131,7 +131,7 @@ ctest:
stage: Test
script:
- cd $OTB_BUILD/ #&& sudo ctest -L OTBTensorflow # Run ctest
- - touch $OTB_TEST_DIR/toto.txt
+ - mkdir -p $OTB_TEST_DIR/testing && touch $OTB_TEST_DIR/testing/toto.txt
after_script:
- cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
--
GitLab
From 459f2b7870807e2f4bfb21ca7fc13f4c8e659804 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Sun, 2 Apr 2023 17:57:42 +0200
Subject: [PATCH 34/72] CI: try to build with C++17
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 98ef447b..c7164d3f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -131,7 +131,7 @@ ctest:
stage: Test
script:
- cd $OTB_BUILD/ #&& sudo ctest -L OTBTensorflow # Run ctest
- - mkdir -p $OTB_TEST_DIR/testing && touch $OTB_TEST_DIR/testing/toto.txt
+ - sudo mkdir -p $OTB_TEST_DIR/testing && sudo touch $OTB_TEST_DIR/testing/toto.txt
after_script:
- cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
--
GitLab
From b29b762bcee2f71dbe2eb3b9e7d8a1d832c8f0db Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 10:20:03 +0200
Subject: [PATCH 35/72] CI: build doc
---
.gitlab-ci.yml | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c7164d3f..6e3dccbb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -28,6 +28,7 @@ workflow:
stages:
- Build
- Static Analysis
+ - Documentation
- Test
- Applications Test
- Update dev image
@@ -119,6 +120,32 @@ cppcheck:
- sudo apt update && sudo apt install cppcheck -y
- cd $OTBTF_SRC/ && cppcheck --enable=all --error-exitcode=1 -I include/ --suppress=missingInclude --suppress=unusedFunction .
+.doc_base:
+ stage: Documentation
+ before_script:
+ - pip install -r doc/doc_requirements.txt
+ artifacts:
+ paths:
+ - public
+ - public_test
+
+pages_test:
+ extends: .doc_base
+ except:
+ - master
+ script:
+ - mkdocs build --site-dir public_test
+
+pages:
+ extends: .doc_base
+ only:
+ - master
+ script:
+ - mkdocs build --site-dir public
+ artifacts:
+ paths:
+ - public
+
.tests_base:
artifacts:
paths:
--
GitLab
From e337f7cb2e59f9b4c7ca37510ccfe77c69b96261 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 13:11:07 +0200
Subject: [PATCH 36/72] ADD: .readthedocs.yams
---
.readthedocs.yaml | 12 ++++++++++++
1 file changed, 12 insertions(+)
create mode 100644 .readthedocs.yaml
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 00000000..719167fe
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,12 @@
+version: 2
+build:
+ os: ubuntu-22.04
+ tools:
+ python: "3.10"
+
+mkdocs:
+ configuration: mkdocs.yml
+
+python:
+ install:
+ - requirements: doc/doc_requirements.txt
\ No newline at end of file
--
GitLab
From 6f46ab43376f5a7b5f354de882beeb217017ed13 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 21:19:06 +0200
Subject: [PATCH 37/72] CI: use bazel cache
---
.gitlab-ci.yml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6e3dccbb..1b21c6eb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,6 +59,7 @@ docker image:
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
"."
- docker push $CACHE_IMAGE_BASE
- >
@@ -72,6 +73,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
@@ -87,6 +89,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
--
GitLab
From f3b835f127dd0fc5f58cc26ddcb9672dc4fc189c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 21:28:39 +0200
Subject: [PATCH 38/72] CI: use bazel cache
---
.gitlab-ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1b21c6eb..877cf5fa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,7 @@ docker image:
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
"."
- docker push $CACHE_IMAGE_BASE
- >
@@ -73,7 +73,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
@@ -89,7 +89,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
--
GitLab
From 97dbd505b03504022469b9ffaa8c82466b3acd76 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 21:32:24 +0200
Subject: [PATCH 39/72] CI: use bazel cache
---
.gitlab-ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 877cf5fa..c065ee90 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,7 @@ docker image:
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=172.18.0.111:9090"
"."
- docker push $CACHE_IMAGE_BASE
- >
@@ -73,7 +73,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=172.18.0.111:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
@@ -89,7 +89,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=172.18.0.111:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
--
GitLab
From 3fa72a2fcf8db792344d7d6f19105379a42e5688 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 21:42:24 +0200
Subject: [PATCH 40/72] CI: use bazel cache
---
.gitlab-ci.yml | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c065ee90..0e6597dc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,6 +35,8 @@ stages:
- Ship
.docker_build_base:
+ variables:
+ FF_NETWORK_PER_BUILD: "true"
allow_failure: false
tags: [godzilla]
image: docker:latest
@@ -59,7 +61,7 @@ docker image:
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=172.18.0.111:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
"."
- docker push $CACHE_IMAGE_BASE
- >
@@ -73,7 +75,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=172.18.0.111:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
@@ -89,7 +91,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=172.18.0.111:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
--
GitLab
From 1b391dc39359b2a172547f464e1f8bb88a7f1549 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 21:52:56 +0200
Subject: [PATCH 41/72] CI: use bazel cache
---
.gitlab-ci.yml | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0e6597dc..b3c52da4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,8 +35,6 @@ stages:
- Ship
.docker_build_base:
- variables:
- FF_NETWORK_PER_BUILD: "true"
allow_failure: false
tags: [godzilla]
image: docker:latest
@@ -56,7 +54,6 @@ docker image:
- >
docker build
--target otbtf-base
- --network="host"
--cache-from $CACHE_IMAGE_BASE
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--
GitLab
From 1ec8d4cfc792271fbd44ba96bf400ca213a5cdf3 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 22:03:52 +0200
Subject: [PATCH 42/72] CI: use bazel cache
---
.gitlab-ci.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b3c52da4..48f7c98c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,6 +54,7 @@ docker image:
- >
docker build
--target otbtf-base
+ --network="gitlab-runner-net"
--cache-from $CACHE_IMAGE_BASE
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--
GitLab
From 092e1993ab8c8d63646de640f81e614bbafffbb0 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 22:24:17 +0200
Subject: [PATCH 43/72] CI: use bazel cache
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 48f7c98c..877cf5fa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,7 +54,7 @@ docker image:
- >
docker build
--target otbtf-base
- --network="gitlab-runner-net"
+ --network="host"
--cache-from $CACHE_IMAGE_BASE
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--
GitLab
From 34297168371b748cd7a6e77b545f4bcf91c8b7ba Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 22:35:20 +0200
Subject: [PATCH 44/72] CI: use bazel cache
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 877cf5fa..c9a96549 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,7 @@ docker image:
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.16.10.20:9090"
"."
- docker push $CACHE_IMAGE_BASE
- >
--
GitLab
From 80725443fc487d53a5afce1bbeb7e0f8d98aa7e9 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 22:37:31 +0200
Subject: [PATCH 45/72] CI: use bazel cache
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c9a96549..5bf0c8dc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -73,7 +73,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.16.10.20:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
--
GitLab
From 7da73c356b8549a788ddfb807c2989f3e51ef461 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 22:40:27 +0200
Subject: [PATCH 46/72] CI: use bazel cache
---
.gitlab-ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5bf0c8dc..02c034cb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,7 @@ docker image:
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.16.10.20:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://godzilla:9090"
"."
- docker push $CACHE_IMAGE_BASE
- >
@@ -73,7 +73,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.16.10.20:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://godzilla:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
@@ -89,7 +89,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://172.18.0.111:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://godzilla:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
--
GitLab
From e36bf5d5d65fc66d5990944b137b0382de0ad045 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 22:45:11 +0200
Subject: [PATCH 47/72] CI: use bazel cache
---
.gitlab-ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 02c034cb..bfb348e7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,7 @@ docker image:
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://godzilla:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://\$BAZELCACHE:9090"
"."
- docker push $CACHE_IMAGE_BASE
- >
@@ -73,7 +73,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://godzilla:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://\$BAZELCACHE:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
@@ -89,7 +89,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://godzilla:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://\$BAZELCACHE:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
--
GitLab
From e18cb6b33f8dc1bc6f130a6faaa5b28eb602083b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 22:52:34 +0200
Subject: [PATCH 48/72] CI: use bazel cache
---
.gitlab-ci.yml | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bfb348e7..7dd0b67c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,6 @@ docker image:
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://\$BAZELCACHE:9090"
"."
- docker push $CACHE_IMAGE_BASE
- >
@@ -73,7 +72,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://\$BAZELCACHE:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
@@ -89,7 +88,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://\$BAZELCACHE:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
"."
# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
--
GitLab
From af1a385e7a7501186ebe856f2f43351170e42e2f Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 22:55:16 +0200
Subject: [PATCH 49/72] CI: use bazel cache
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7dd0b67c..6d9e1928 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -72,7 +72,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
@@ -88,7 +88,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
"."
# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
--
GitLab
From 8124894ca3d4abc0924dd5c79c9d3e982d262812 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 22:59:03 +0200
Subject: [PATCH 50/72] CI: use bazel cache
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6d9e1928..7dd0b67c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -72,7 +72,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
"."
# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
@@ -88,7 +88,7 @@ docker image:
--build-arg BZL_CONFIGS=""
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=http://bazelcache:9090"
+ --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
"."
# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
--
GitLab
From db223ede3459ff328e22fcfb9d1210365fa8b056 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Apr 2023 23:01:38 +0200
Subject: [PATCH 51/72] CI: use bazel cache
---
.gitlab-ci.yml | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7dd0b67c..7b13c9fc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,7 +54,6 @@ docker image:
- >
docker build
--target otbtf-base
- --network="host"
--cache-from $CACHE_IMAGE_BASE
--tag $CACHE_IMAGE_BASE
--build-arg BASE_IMG=$CPU_BASE_IMG
@@ -64,7 +63,6 @@ docker image:
- >
docker build
--target builder
- --network="host"
--cache-from $CACHE_IMAGE_BASE
--cache-from $CACHE_IMAGE_BUILDER
--tag $CACHE_IMAGE_BUILDER
@@ -78,7 +76,6 @@ docker image:
- docker push $CACHE_IMAGE_BUILDER
- >
docker build
- --network="host"
--cache-from $CACHE_IMAGE_BASE
--cache-from $CACHE_IMAGE_BUILDER
--cache-from $BRANCH_IMAGE
--
GitLab
From 4b2e66829dc20f8a07388ba535983f0a064f70a8 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 08:33:12 +0200
Subject: [PATCH 52/72] CI: build without --host
---
.gitlab-ci.yml | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7b13c9fc..da9aee6d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -229,10 +229,10 @@ deploy_cpu:
DOCKERHUB_LATEST: $DOCKERHUB_BASE:latest
script:
# cpu
- - docker build --network='host' --tag $IMAGE_CPU --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" .
+ - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_CPU --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" .
- docker push $IMAGE_CPU
# cpu-dev
- - docker build --network='host' --tag $IMAGE_CPUDEV --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" --build-arg KEEP_SRC_OTB=true .
+ - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_CPUDEV --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" --build-arg KEEP_SRC_OTB=true .
- docker push $IMAGE_CPUDEV
# push images on dockerhub
- echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
@@ -255,16 +255,16 @@ deploy_gpu:
DOCKERHUB_GPUDEV: $DOCKERHUB_IMAGE_BASE-gpu-dev
script:
# gpu-opt
- - docker build --network='host' --tag $IMAGE_GPUOPT --build-arg BASE_IMG=$GPU_BASE_IMG .
+ - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPUOPT --build-arg BASE_IMG=$GPU_BASE_IMG .
- docker push $IMAGE_GPUOPT
# gpu-opt-dev
- - docker build --network='host' --tag $IMAGE_GPUOPTDEV --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
+ - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPUOPTDEV --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
- docker push $IMAGE_GPUOPTDEV
# gpu-basic
- - docker build --network='host' --tag $IMAGE_GPU --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg BZL_CONFIGS="" .
+ - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPU --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg BZL_CONFIGS="" .
- docker push $IMAGE_GPU
# gpu-basic-dev
- - docker build --network='host' --tag $IMAGE_GPUDEV --build-arg BZL_CONFIGS="" --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
+ - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPUDEV --build-arg BZL_CONFIGS="" --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
- docker push $IMAGE_GPUDEV
# push gpu-basic* images on dockerhub
- echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
--
GitLab
From 0cf9999db26628e39875faf8c6f588d8f26136b5 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 08:33:29 +0200
Subject: [PATCH 53/72] DOC: hack git-lfs issue
---
doc/gen_ref_pages.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/doc/gen_ref_pages.py b/doc/gen_ref_pages.py
index a02b8180..93fc1b4b 100644
--- a/doc/gen_ref_pages.py
+++ b/doc/gen_ref_pages.py
@@ -23,3 +23,11 @@ for path in sorted(Path("otbtf").rglob("*.py")): #
mkdocs_gen_files.set_edit_path(full_doc_path, path)
+# Workaround to install and execute git-lfs on Read the Docs
+import os
+if not os.path.exists('./git-lfs'):
+ os.system('wget https://github.com/git-lfs/git-lfs/releases/download/v2.7.1/git-lfs-linux-amd64-v2.7.1.tar.gz')
+ os.system('tar xvfz git-lfs-linux-amd64-v2.7.1.tar.gz')
+ os.system('./git-lfs install') # make lfs available in current repository
+ os.system('./git-lfs fetch') # download content from remote
+ os.system('./git-lfs checkout') # make local files to have the real content on them
--
GitLab
From 50e58e8dc41d248b36005da258f8c7afe77c9148 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:01:14 +0200
Subject: [PATCH 54/72] COMP: set version to 4.0.0
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 9dd65a95..958be96a 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setuptools.setup(
name="otbtf",
- version="3.5.0",
+ version="4.0.0",
author="Remi Cresson",
author_email="remi.cresson@inrae.fr",
description="OTBTF: Orfeo ToolBox meets TensorFlow",
--
GitLab
From 405d53ba9d2a7fe4cac3f1a19ff45d7170269c10 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:01:37 +0200
Subject: [PATCH 55/72] COMP: hack to build OTB with C++17 with tests
---
Dockerfile | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 280eb639..372ae83c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -108,7 +108,20 @@ RUN apt-get update -y \
&& apt-get install --reinstall ca-certificates -y \
&& update-ca-certificates \
&& git clone https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git \
- && cd otb && git checkout $OTB && sed -i 's/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g' CMakeLists.txt && cd .. \
+ && cd otb && git checkout $OTB \
+# <---------------------------------------- Begin dirty hack
+# This is a dirty hack for release 4.0.0alpha
+# We have to wait that OTB moves from C++14 to C++17
+# See https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2338
+ && sed -i 's/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g' CMakeLists.txt \
+ && echo "" > Modules/Filtering/ImageManipulation/test/CMakeLists.txt
+ && echo "" > Modules/Segmentation/Conversion/test/CMakeLists.txt \
+ && echo "" > Modules/Radiometry/Indices/test/CMakeLists.txt \
+ && echo "" > Modules/Learning/DempsterShafer/test/CMakeLists.txt \
+ && echo "" > Modules/Feature/Edge/test/CMakeLists.txt \
+ && echo "" > Modules/Core/ImageBase/test/CMakeLists.txt \
+# <---------------------------------------- End dirty hack
+ && cd .. \
&& mkdir -p build \
&& cd build \
&& if $OTBTESTS; then \
@@ -165,7 +178,8 @@ ENV PYTHONPATH="/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python"
ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
RUN pip install -e /src/otbtf
-# Default user, directory and command (bash is the entrypoint when using 'docker create')
+# Default user, directory and command (bash is the entrypoint when using
+# 'docker create')
RUN useradd -s /bin/bash -m otbuser
WORKDIR /home/otbuser
@@ -175,7 +189,8 @@ RUN if $SUDO; then \
usermod -a -G sudo otbuser \
&& echo "otbuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers; fi
-# Set /src/otbtf ownership to otbuser (but you still need 'sudo -i' in order to rebuild TF or OTB)
+# Set /src/otbtf ownership to otbuser (but you still need 'sudo -i' in order
+# to rebuild TF or OTB)
RUN chown -R otbuser:otbuser /src/otbtf
# This won't prevent ownership problems with volumes if you're not UID 1000
--
GitLab
From 69d279d1b3c736377893c04b251a6b7329af2f27 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:01:56 +0200
Subject: [PATCH 56/72] CI: reactivate OTB testing
---
.gitlab-ci.yml | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index da9aee6d..5efafd58 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
variables:
- OTBTF_VERSION: 3.5.0
+ OTBTF_VERSION: 4.0.0alpha
OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory
OTBTF_SRC: /src/otbtf # Local OTBTF source directory
OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory
@@ -71,8 +71,8 @@ docker image:
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
--build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
+ --build-arg OTBTESTS="true"
"."
-# --build-arg OTBTESTS="true"
- docker push $CACHE_IMAGE_BUILDER
- >
docker build
@@ -86,8 +86,8 @@ docker image:
--build-arg BASE_IMG=$CPU_BASE_IMG
--build-arg BUILDKIT_INLINE_CACHE=1
--build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
+ --build-arg OTBTESTS="true"
"."
-# --build-arg OTBTESTS="true"
- docker push $BRANCH_IMAGE
.static_analysis_base:
@@ -156,8 +156,7 @@ ctest:
extends: .tests_base
stage: Test
script:
- - cd $OTB_BUILD/ #&& sudo ctest -L OTBTensorflow # Run ctest
- - sudo mkdir -p $OTB_TEST_DIR/testing && sudo touch $OTB_TEST_DIR/testing/toto.txt
+ - cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow # Run ctest
after_script:
- cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
--
GitLab
From fec1d38fd75cffe359df5fae2e2882539cfc247a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:02:09 +0200
Subject: [PATCH 57/72] ADD: __version__
---
otbtf/__init__.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/otbtf/__init__.py b/otbtf/__init__.py
index 70afdf04..04ac11db 100644
--- a/otbtf/__init__.py
+++ b/otbtf/__init__.py
@@ -20,6 +20,7 @@
"""
OTBTF python module
"""
+import pkg_resources
try:
from otbtf.utils import read_as_np_arr, gdal_open # noqa
from otbtf.dataset import Buffer, PatchesReaderBase, PatchesImagesReader, \
@@ -32,3 +33,4 @@ except ImportError:
from otbtf.tfrecords import TFRecords # noqa
from otbtf.model import ModelBase # noqa
+__version__ = pkg_resources.require("otbtf")[0].version
--
GitLab
From 41349d1e55afe39267b27cf721bbe6af5f86e0f0 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:02:26 +0200
Subject: [PATCH 58/72] DOC: add doc badge
---
README.md | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/README.md b/README.md
index 64dc851d..486087a9 100644
--- a/README.md
+++ b/README.md
@@ -9,24 +9,24 @@
<a href="https://gitlab.irstea.fr/remi.cresson/otbtf/-/commits/master">
<img src="https://gitlab.irstea.fr/remi.cresson/otbtf/badges/master/pipeline.svg">
</a>
+<img src='https://readthedocs.org/projects/otbtf/badge/?version=latest' alt='Documentation Status' />
<a href="LICENSE">
<img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg">
</a>
</p>
OTBTF is a remote module of the [Orfeo ToolBox](https://www.orfeo-toolbox.org).
-It provides a generic, multi purpose deep learning framework, targeting remote
+It provides a generic, multi-purpose deep learning framework, targeting remote
sensing images processing. It contains a set of new process objects for OTB
-that internally invoke [Tensorflow](https://www.tensorflow.org/), and new [OTB
-applications](#otb-applications) to perform deep learning with real-world
-remote sensing images. Applications can be used to build OTB pipelines from
-Python or C++ APIs. OTBTF also includes a [python API](#python-api) to build
-quickly Keras compliant models suited for remote sensing imagery, easy to
-train in distributed environments.
+that internally invoke [Tensorflow](https://www.tensorflow.org/), and new OTB
+applications to perform deep learning with real-world remote sensing images.
+Applications can be used to build OTB pipelines from Python or C++ APIs. OTBTF
+also includes a python API to build quickly Keras compliant models suited for
+remote sensing imagery, easy to train in distributed environments.
## Documentation
-[otbtf.readthedocs.io](otbtf.readthedocs.io)
+The documentation is available on [otbtf.readthedocs.io](otbtf.readthedocs.io).
## Use
--
GitLab
From a3d5663e0bc6b897c0808cebc0b9c908fc049914 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:08:01 +0200
Subject: [PATCH 59/72] ADD: a new test to reproduce the old bug with tf/otb
imports together
---
test/imports_test.py | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
create mode 100644 test/imports_test.py
diff --git a/test/imports_test.py b/test/imports_test.py
new file mode 100644
index 00000000..a89ab3dc
--- /dev/null
+++ b/test/imports_test.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import pytest
+import unittest
+
+class ImportsTest(unittest.TestCase):
+
+ def test_import_both1(self):
+ import tensorflow
+ self.assertTrue(tensorflow.__version__)
+ import otbApplication
+ self.assertTrue(otbApplication.Registry_GetAvailableApplications())
+
+
+ def test_import_both2(self):
+ import otbApplication
+ self.assertTrue(otbApplication.Registry_GetAvailableApplications())
+ import tensorflow
+ self.assertTrue(tensorflow.__version__)
+
+
+if __name__ == '__main__':
+ unittest.main()
--
GitLab
From f29cf7e7bcde74247c47c4f3ccbf7491668c96b7 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:10:22 +0200
Subject: [PATCH 60/72] CI: add the new python imports test
---
.gitlab-ci.yml | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5efafd58..1d4d97fa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -202,6 +202,11 @@ geos_enabled:
script:
- python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_geos_enabled.xml $OTBTF_SRC/test/geos_test.py
+imports:
+ extends: .applications_test_base
+ script:
+ - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_imports.xml $OTBTF_SRC/test/imports_test.py
+
deploy_cpu-dev-testing:
stage: Update dev image
extends: .docker_build_base
--
GitLab
From 610b4b8361d4d3ffe122b47b7d5ab3335e13c15e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:13:48 +0200
Subject: [PATCH 61/72] ADD: hack to compile OTB with c++17
---
Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 372ae83c..f8b82f79 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -114,7 +114,7 @@ RUN apt-get update -y \
# We have to wait that OTB moves from C++14 to C++17
# See https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2338
&& sed -i 's/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g' CMakeLists.txt \
- && echo "" > Modules/Filtering/ImageManipulation/test/CMakeLists.txt
+ && echo "" > Modules/Filtering/ImageManipulation/test/CMakeLists.txt \
&& echo "" > Modules/Segmentation/Conversion/test/CMakeLists.txt \
&& echo "" > Modules/Radiometry/Indices/test/CMakeLists.txt \
&& echo "" > Modules/Learning/DempsterShafer/test/CMakeLists.txt \
--
GitLab
From a6267783944757695c3059e6670e1ad9e37665a8 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:27:04 +0200
Subject: [PATCH 62/72] STY: line breaks
---
doc/custom_theme/main.html | 5 -----
tools/docker/build-env-tf.sh | 6 ++++--
2 files changed, 4 insertions(+), 7 deletions(-)
delete mode 100644 doc/custom_theme/main.html
diff --git a/doc/custom_theme/main.html b/doc/custom_theme/main.html
deleted file mode 100644
index d9b3e888..00000000
--- a/doc/custom_theme/main.html
+++ /dev/null
@@ -1,5 +0,0 @@
-{% extends "base.html" %}
-
-{% block extrahead %}
-<meta name="google-site-verification" content="lKzwW-jjP466yU5s9vwERWKm0DHBXWmH1sIN2sas7sQ" />
-{% endblock %}
\ No newline at end of file
diff --git a/tools/docker/build-env-tf.sh b/tools/docker/build-env-tf.sh
index b29f6c1a..ff5c5692 100644
--- a/tools/docker/build-env-tf.sh
+++ b/tools/docker/build-env-tf.sh
@@ -1,7 +1,9 @@
### TF - bazel build env variables
-# As in official TF wheels, you'll need to remove "-march=native" to ensure portability (avoid AVX2 / AVX512 compatibility issues)
-# You could also add CPUs instructions one by one, in this example to avoid only AVX512 but enable commons optimizations like FMA, SSE4.2 and AVX2
+# As in official TF wheels, you'll need to remove "-march=native" to ensure
+# portability (avoid AVX2 / AVX512 compatibility issues)
+# You could also add CPUs instructions one by one, in this example to avoid
+# only AVX512 but enable commons optimizations like FMA, SSE4.2 and AVX2
#export CC_OPT_FLAGS="-Wno-sign-compare --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-mfpmath=both --copt=-msse4.2"
export CC_OPT_FLAGS="-march=native -Wno-sign-compare"
export GCC_HOST_COMPILER_PATH=$(which gcc)
--
GitLab
From 15cfb410684593b7681676fa1e3b98133e1f7ed0 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:27:29 +0200
Subject: [PATCH 63/72] COMP: remove deprecated OTB compilation flags
---
tools/docker/build-flags-otb.txt | 13 -------------
1 file changed, 13 deletions(-)
diff --git a/tools/docker/build-flags-otb.txt b/tools/docker/build-flags-otb.txt
index fe0eb08d..95b8462c 100644
--- a/tools/docker/build-flags-otb.txt
+++ b/tools/docker/build-flags-otb.txt
@@ -6,9 +6,6 @@
-DUSE_SYSTEM_GDAL=OFF
-DUSE_SYSTEM_GEOS=ON
-DUSE_SYSTEM_GEOTIFF=OFF
--DUSE_SYSTEM_GLEW=ON
--DUSE_SYSTEM_GLFW=ON
--DUSE_SYSTEM_GLUT=ON
-DUSE_SYSTEM_GSL=ON
-DUSE_SYSTEM_ITK=ON
-DUSE_SYSTEM_LIBKML=ON
@@ -17,16 +14,6 @@
-DUSE_SYSTEM_MUPARSERX=ON
-DUSE_SYSTEM_OPENCV=ON
-DUSE_SYSTEM_PNG=ON
--DUSE_SYSTEM_QT5=ON
--DUSE_SYSTEM_QWT=ON
-DUSE_SYSTEM_TINYXML=ON
-DUSE_SYSTEM_ZLIB=ON
-DUSE_SYSTEM_SWIG=ON
-
--DOTB_USE_QT=OFF
--DOTB_USE_OPENGL=OFF
--DOTB_USE_GLUT=OFF
--DOTB_USE_GLEW=OFF
--DOTB_USE_GLFW=OFF
-
--DGDAL_SB_EXTRA_OPTIONS=--with-geos
--
GitLab
From 767f05e6276a77b05b700a824cdc00d335d6a00e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:27:50 +0200
Subject: [PATCH 64/72] STY: line breaks
---
tools/docker/multibuild.sh | 104 +++++++++++++++++++++++++------------
1 file changed, 71 insertions(+), 33 deletions(-)
diff --git a/tools/docker/multibuild.sh b/tools/docker/multibuild.sh
index c88bb0ac..9373d292 100644
--- a/tools/docker/multibuild.sh
+++ b/tools/docker/multibuild.sh
@@ -1,37 +1,75 @@
#!/bin/bash
-### Docker multibuild and push, see default args and more examples in tools/docker/README.md
-RELEASE=2.5
-UBUNTU=20.04
-CUDA=11.2.2
-CUDNN=8
-IMG=ubuntu:$UBUNTU
-GPU_IMG=nvidia/cuda:$CUDA-cudnn$CUDNN-devel-ubuntu$UBUNTU
+# Various docker builds using bazel cache
+RELEASE=3.5
+CPU_IMG=ubuntu:22.04
+GPU_IMG=nvidia/cuda:12.1.0-devel-ubuntu22.04
## Bazel remote cache daemon
mkdir -p $HOME/.cache/bazel-remote
-docker run -d -u 1000:1000 -v $HOME/.cache/bazel-remote:/data -p 9090:8080 buchgr/bazel-remote-cache --max_size=20
-
-### CPU (no MKL)
-docker build --network='host' -t mdl4eo/otbtf$RELEASE:cpu-dev --build-arg BASE_IMG=$IMG --build-arg KEEP_SRC_OTB=true .
-docker build --network='host' -t mdl4eo/otbtf$RELEASE:cpu --build-arg BASE_IMG=$IMG .
-#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-cpu-gui --build-arg BASE_IMG=$IMG --build-arg GUI=true .
-
-### MKL is enabled with bazel config flag
-#MKL_CONF="--config=nogcp --config=noaws --config=nohdfs --config=mkl --config=opt"
-#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-cpu-mkl --build-arg BASE_IMG=$IMG --build-arg BZL_CONFIGS="$MKL_CONF" .
-#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-cpu-mkl-dev --build-arg BASE_IMG=$IMG --build-arg BZL_CONFIGS="$MKL_CONF" --build-arg KEEP_SRC_OTB=true .
-
-### GPU support is enabled if CUDA is found in /usr/local
-docker build --network='host' -t mdl4eo/otbtf$RELEASE:gpu-dev --build-arg BASE_IMG=$GPU_IMG --build-arg KEEP_SRC_OTB=true .
-docker build --network='host' -t mdl4eo/otbtf$RELEASE:gpu --build-arg BASE_IMG=$GPU_IMG .
-#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-gpu-gui --build-arg BASE_IMG=$GPU_IMG --build-arg GUI=true .
-
-#docker login
-docker push mdl4eo/otbtf$RELEASE:-cpu-dev
-docker push mdl4eo/otbtf$RELEASE:-cpu
-#docker push mdl4eo/otbtf$RELEASE:-cpu-gui
-#docker push mdl4eo/otbtf$RELEASE:-cpu-mkl
-
-docker push mdl4eo/otbtf$RELEASE:-gpu-dev
-docker push mdl4eo/otbtf$RELEASE:-gpu
-#docker push mdl4eo/otbtf$RELEASE:-gpu-gui
+docker run -d -u 1000:1000 \
+-v $HOME/.cache/bazel-remote:/data \
+-p 9090:8080 \
+buchgr/bazel-remote-cache --max_size=20
+
+### CPU images
+
+# CPU-Dev
+docker build . \
+--network='host' \
+-t mdl4eo/otbtf:$RELEASE-cpu-dev \
+--build-arg BASE_IMG=$CPU_IMG \
+--build-arg KEEP_SRC_OTB=true
+
+# CPU
+docker build . \
+--network='host' \
+-t mdl4eo/otbtf:$RELEASE-cpu \
+--build-arg BASE_IMG=$CPU_IMG
+
+# CPU-GUI
+docker build . \
+--network='host' \
+-t mdl4eo/otbtf:$RELEASE-cpu-gui \
+--build-arg BASE_IMG=$CPU_IMG \
+--build-arg GUI=true
+
+### CPU images with Intel MKL support
+MKL_CONF="--config=nogcp --config=noaws --config=nohdfs --config=mkl --config=opt"
+
+# CPU-MKL
+docker build . \
+--network='host' \
+-t mdl4eo/otbtf:$RELEASE-cpu-mkl \
+--build-arg BASE_IMG=$CPU_IMG \
+--build-arg BZL_CONFIGS="$MKL_CONF"
+
+# CPU-MKL-Dev
+docker build . \
+--network='host' \
+-t mdl4eo/otbtf:$RELEASE-cpu-mkl-dev \
+--build-arg BASE_IMG=$CPU_IMG \
+--build-arg BZL_CONFIGS="$MKL_CONF" \
+--build-arg KEEP_SRC_OTB=true
+
+### GPU enabled images
+# Support is enabled if CUDA is found in /usr/local
+
+# GPU
+docker build . \
+--network='host' \
+-t mdl4eo/otbtf:$RELEASE-gpu-dev \
+--build-arg BASE_IMG=$GPU_IMG \
+--build-arg KEEP_SRC_OTB=true
+
+# GPU-Dev
+docker build . \
+--network='host' \
+-t mdl4eo/otbtf:$RELEASE-gpu \
+--build-arg BASE_IMG=$GPU_IMG
+
+# GPU-GUI
+docker build . \
+--network='host' \
+-t mdl4eo/otbtf:$RELEASE-gpu-gui \
+--build-arg BASE_IMG=$GPU_IMG \
+--build-arg GUI=true
--
GitLab
From 95416ef15c10f50a809fea7eab97e29a2527552e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:28:15 +0200
Subject: [PATCH 65/72] DOC: update python API tutorial
---
doc/api_tutorial.md | 4 ----
1 file changed, 4 deletions(-)
diff --git a/doc/api_tutorial.md b/doc/api_tutorial.md
index 443cbc4f..bcd6ea45 100644
--- a/doc/api_tutorial.md
+++ b/doc/api_tutorial.md
@@ -331,10 +331,6 @@ Finally we can save our model as a SavedModel:
model.save("/tmp/my_1st_savedmodel")
```
-The whole code is available in the *otbtf/examples/tensorflow_v2x/fcnn*
-directory.
-[`ModelBase.__init__()`](reference/otbtf/examples/tensorflow_v2x/fcnn/__init__.html)
-
## Inference
This section show how to apply the fully convolutional model over an entire
--
GitLab
From ee9d6efa4ca108f1ad99324a2b7c0b81005baef2 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:28:36 +0200
Subject: [PATCH 66/72] COMP: cleaning mkdocs cfg
---
mkdocs.yml | 5 -----
1 file changed, 5 deletions(-)
diff --git a/mkdocs.yml b/mkdocs.yml
index 9c3b7a0f..bef032e1 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -4,16 +4,11 @@ theme:
logo: images/logo.png
icon:
repo: fontawesome/brands/github
-# admonition:
-# <type>: <icon>
features:
- navigation.instant
- content.code.copy
- content.code.annotate
- toc.follow
-# analytics:
-# - gtag: G-PD85X2X108
-# custom_dir: doc/custom_theme
plugins:
- search
--
GitLab
From c4c5626d6f8e74189d314877ee578186e466b932 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:28:50 +0200
Subject: [PATCH 67/72] DOC: update readme
---
README.md | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 486087a9..b51317c6 100644
--- a/README.md
+++ b/README.md
@@ -30,13 +30,15 @@ The documentation is available on [otbtf.readthedocs.io](otbtf.readthedocs.io).
## Use
-We recommend to use our latest GPU enabled docker images.
+You can use our latest GPU enabled docker images.
```bash
docker run --runtime=nvidia -ti mdl4eo/otbtf:3.4.0-gpu otbcli_PatchesExtraction
docker run --runtime=nvidia -ti mdl4eo/otbtf:3.4.0-gpu python -c "import otbtf"
```
+You can also build OTBTF from sources (see the documentation)
+
## Cite
```
--
GitLab
From e02f56728be8a6284ecd53cf98c1839349df70db Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:29:11 +0200
Subject: [PATCH 68/72] ADD: release notes for 4.0.0alpha
---
RELEASE_NOTES.txt | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index b96ba1ac..7cf3b01d 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,12 +1,17 @@
-Version 4.0.0 (3 apr 2023)
-----------------------------------------------------------------
-* Big improvement of the documentation, now have a readthedocs.io for otbtf !
+Version 4.0.0alpha (4 apr 2023)
+----------------------------------------------------------------
+* Big improvement of the documentation:
+ - Re-structure the entire doc, remove deprecated stuff, etc.
+ - Add a nice mkdocs template
+ - Docs are now hosted at otbtf.readthedocs.io
+ - Add a new section on the python API (end-to-end tutorial)
+ - A lot of new sections: distributed training, etc...
* Refactoring all the python classes to enforce pep8
* Ubuntu version: 22.04
* Cuda version: 12.1.0
* Tensorflow version: 2.12.0
-* OTB version: 5086d7601d80f2427f4d4d7f2398ec46e7efa300 (pre-9.0beta)
-* Fixed Tensorflow error "Cannot register 2 metrics with the same name"
+* Fixed Tensorflow error "Cannot register 2 metrics with the same name" + new test
+* Faster CI build thanks to bazel remote cache
Version 3.4.0 (22 mar 2023)
----------------------------------------------------------------
--
GitLab
From 3758e67d8f05198b52b3f9673d01b5f705763792 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 10:49:53 +0200
Subject: [PATCH 69/72] DOC: add release 4.0.0alpha in documentation
---
README.md | 4 ++--
doc/docker_use.md | 32 +++++++++++++++++++-------------
2 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/README.md b/README.md
index b51317c6..fb2345cc 100644
--- a/README.md
+++ b/README.md
@@ -33,8 +33,8 @@ The documentation is available on [otbtf.readthedocs.io](otbtf.readthedocs.io).
You can use our latest GPU enabled docker images.
```bash
-docker run --runtime=nvidia -ti mdl4eo/otbtf:3.4.0-gpu otbcli_PatchesExtraction
-docker run --runtime=nvidia -ti mdl4eo/otbtf:3.4.0-gpu python -c "import otbtf"
+docker run --runtime=nvidia -ti mdl4eo/otbtf:4.0.0alpha-gpu otbcli_PatchesExtraction
+docker run --runtime=nvidia -ti mdl4eo/otbtf:4.0.0alpha-gpu python -c "import otbtf"
```
You can also build OTBTF from sources (see the documentation)
diff --git a/doc/docker_use.md b/doc/docker_use.md
index 01e75bf0..949926c7 100644
--- a/doc/docker_use.md
+++ b/doc/docker_use.md
@@ -5,13 +5,13 @@ We recommend to use OTBTF from official docker images.
Latest CPU-only docker image:
```commandline
-docker pull mdl4eo/otbtf:3.4.0-cpu
+docker pull mdl4eo/otbtf:4.0.0alpha-cpu
```
Latest GPU-ready docker image:
```commandline
-docker pull mdl4eo/otbtf:3.4.0-gpu
+docker pull mdl4eo/otbtf:4.0.0alpha-gpu
```
Read more in the following sections.
@@ -23,14 +23,14 @@ Here is the list of the latest OTBTF docker images hosted on
Since OTBTF >= 3.2.1 you can find the latest docker images on
[gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry).
-| Name | Os | TF | OTB | Description | Dev files | Compute capability |
-|------------------------------------------------------------------------------------| ------------- | ------ |-------| ---------------------- | --------- | ------------------ |
-| **mdl4eo/otbtf:3.4.0-cpu** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.4.0-cpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.4.0-gpu** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.4.0-gpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.4.0-gpu-opt** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.4.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| Name | Os | TF | OTB | Description | Dev files | Compute capability |
+|------------------------------------------------------------------------------------| ------------- |-------|-------| ---------------------- | --------- | ------------------ |
+| **mdl4eo/otbtf:4.0.0alpha-cpu** | Ubuntu Focal | r2.12 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0alpha-cpu-dev** | Ubuntu Focal | r2.12 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0alpha-gpu** | Ubuntu Focal | r2.12 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0alpha-gpu-dev** | Ubuntu Focal | r2.12 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0alpha-gpu-opt** | Ubuntu Focal | r2.12 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0alpha-gpu-opt-dev** | Ubuntu Focal | r2.12 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
The list of older releases is available [here](#older-images).
@@ -51,13 +51,13 @@ You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime*
With Docker version earlier than 19.03 :
```bash
-docker run --runtime=nvidia -ti mdl4eo/otbtf:3.4.0-gpu bash
+docker run --runtime=nvidia -ti mdl4eo/otbtf:4.0.0alpha-gpu bash
```
With Docker version including and after 19.03 :
```bash
-docker run --gpus all -ti mdl4eo/otbtf:3.4.0-gpu bash
+docker run --gpus all -ti mdl4eo/otbtf:4.0.0alpha-gpu bash
```
You can find some details on the **GPU docker image** and some **docker tips
@@ -80,7 +80,7 @@ See here how to install docker on Ubuntu
1. Install [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10#manual-installation-steps) (Windows Subsystem for Linux)
2. Install [docker desktop](https://www.docker.com/products/docker-desktop)
3. Start **docker desktop** and **enable WSL2** from *Settings* > *General* then tick the box *Use the WSL2 based engine*
-3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf:3.4.0-cpu`
+3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf:4.0.0alpha-cpu`
4. Open **docker desktop**, and check that the docker is running in the **Container/Apps** menu
5. From **docker desktop**, click on the icon highlighted as shown below, and use the bash terminal that should pop up!
@@ -152,4 +152,10 @@ Here you can find the list of older releases of OTBTF:
| **mdl4eo/otbtf:3.3.3-gpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.3-gpu-opt** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.3-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.4.0-cpu** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.4.0-cpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.4.0-gpu** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.4.0-gpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.4.0-gpu-opt** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.4.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
--
GitLab
From 48207f26813201df8c0f751313edcbef444843d1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 12:06:05 +0200
Subject: [PATCH 70/72] CI: bump version to 4.0.0
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1d4d97fa..596e7ed1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
variables:
- OTBTF_VERSION: 4.0.0alpha
+ OTBTF_VERSION: 4.0.0
OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory
OTBTF_SRC: /src/otbtf # Local OTBTF source directory
OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory
--
GitLab
From 5459e465200dacc9dd9e7e5e55d4e59f8d34cc50 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 12:06:20 +0200
Subject: [PATCH 71/72] DOC: update with version 4.0.0
---
README.md | 4 ++--
doc/docker_use.md | 22 +++++++++++-----------
2 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/README.md b/README.md
index fb2345cc..b61be7b9 100644
--- a/README.md
+++ b/README.md
@@ -33,8 +33,8 @@ The documentation is available on [otbtf.readthedocs.io](otbtf.readthedocs.io).
You can use our latest GPU enabled docker images.
```bash
-docker run --runtime=nvidia -ti mdl4eo/otbtf:4.0.0alpha-gpu otbcli_PatchesExtraction
-docker run --runtime=nvidia -ti mdl4eo/otbtf:4.0.0alpha-gpu python -c "import otbtf"
+docker run --runtime=nvidia -ti mdl4eo/otbtf:4.0.0-gpu otbcli_PatchesExtraction
+docker run --runtime=nvidia -ti mdl4eo/otbtf:4.0.0-gpu python -c "import otbtf"
```
You can also build OTBTF from sources (see the documentation)
diff --git a/doc/docker_use.md b/doc/docker_use.md
index 949926c7..ba3c083d 100644
--- a/doc/docker_use.md
+++ b/doc/docker_use.md
@@ -5,13 +5,13 @@ We recommend to use OTBTF from official docker images.
Latest CPU-only docker image:
```commandline
-docker pull mdl4eo/otbtf:4.0.0alpha-cpu
+docker pull mdl4eo/otbtf:4.0.0-cpu
```
Latest GPU-ready docker image:
```commandline
-docker pull mdl4eo/otbtf:4.0.0alpha-gpu
+docker pull mdl4eo/otbtf:4.0.0-gpu
```
Read more in the following sections.
@@ -25,12 +25,12 @@ Since OTBTF >= 3.2.1 you can find the latest docker images on
| Name | Os | TF | OTB | Description | Dev files | Compute capability |
|------------------------------------------------------------------------------------| ------------- |-------|-------| ---------------------- | --------- | ------------------ |
-| **mdl4eo/otbtf:4.0.0alpha-cpu** | Ubuntu Focal | r2.12 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.0.0alpha-cpu-dev** | Ubuntu Focal | r2.12 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.0.0alpha-gpu** | Ubuntu Focal | r2.12 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.0.0alpha-gpu-dev** | Ubuntu Focal | r2.12 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0alpha-gpu-opt** | Ubuntu Focal | r2.12 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0alpha-gpu-opt-dev** | Ubuntu Focal | r2.12 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0-cpu** | Ubuntu Focal | r2.12 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0-cpu-dev** | Ubuntu Focal | r2.12 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0-gpu** | Ubuntu Focal | r2.12 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0-gpu-dev** | Ubuntu Focal | r2.12 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0-gpu-opt** | Ubuntu Focal | r2.12 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0-gpu-opt-dev** | Ubuntu Focal | r2.12 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
The list of older releases is available [here](#older-images).
@@ -51,13 +51,13 @@ You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime*
With Docker version earlier than 19.03 :
```bash
-docker run --runtime=nvidia -ti mdl4eo/otbtf:4.0.0alpha-gpu bash
+docker run --runtime=nvidia -ti mdl4eo/otbtf:4.0.0-gpu bash
```
With Docker version including and after 19.03 :
```bash
-docker run --gpus all -ti mdl4eo/otbtf:4.0.0alpha-gpu bash
+docker run --gpus all -ti mdl4eo/otbtf:4.0.0-gpu bash
```
You can find some details on the **GPU docker image** and some **docker tips
@@ -80,7 +80,7 @@ See here how to install docker on Ubuntu
1. Install [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10#manual-installation-steps) (Windows Subsystem for Linux)
2. Install [docker desktop](https://www.docker.com/products/docker-desktop)
3. Start **docker desktop** and **enable WSL2** from *Settings* > *General* then tick the box *Use the WSL2 based engine*
-3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf:4.0.0alpha-cpu`
+3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf:4.0.0-cpu`
4. Open **docker desktop**, and check that the docker is running in the **Container/Apps** menu
5. From **docker desktop**, click on the icon highlighted as shown below, and use the bash terminal that should pop up!
--
GitLab
From a298e947a5c1dcfcfaa6e0378c4a6cf0355e3911 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 4 Apr 2023 12:29:01 +0200
Subject: [PATCH 72/72] DOC: update docker image ubuntu version
---
doc/docker_use.md | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/doc/docker_use.md b/doc/docker_use.md
index ba3c083d..47749325 100644
--- a/doc/docker_use.md
+++ b/doc/docker_use.md
@@ -25,12 +25,12 @@ Since OTBTF >= 3.2.1 you can find the latest docker images on
| Name | Os | TF | OTB | Description | Dev files | Compute capability |
|------------------------------------------------------------------------------------| ------------- |-------|-------| ---------------------- | --------- | ------------------ |
-| **mdl4eo/otbtf:4.0.0-cpu** | Ubuntu Focal | r2.12 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.0.0-cpu-dev** | Ubuntu Focal | r2.12 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.0.0-gpu** | Ubuntu Focal | r2.12 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.0.0-gpu-dev** | Ubuntu Focal | r2.12 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0-gpu-opt** | Ubuntu Focal | r2.12 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0-gpu-opt-dev** | Ubuntu Focal | r2.12 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0-cpu** | Ubuntu Jammy | r2.12 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0-cpu-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0-gpu** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.0.0-gpu-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0-gpu-opt** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0-gpu-opt-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
The list of older releases is available [here](#older-images).
@@ -95,8 +95,10 @@ Troubleshooting:
Some users have reported to use OTBTF with GPU in windows 10 using WSL2.
How to install WSL2 with Cuda on windows 10:
- https://docs.nvidia.com/cuda/wsl-user-guide/index.html
- https://docs.docker.com/docker-for-windows/wsl/#gpu-support
+
+ - [WSL user guide](https://docs.nvidia.com/cuda/wsl-user-guide/index.html)
+ - [XSL GPU support](https://docs.docker.com/docker-for-windows/wsl/#gpu-support)
+
## Build your own images
--
GitLab