Merge pull request #22 from MateusRoder/maxpool

Optimizing DBN training

examples/applications/deep/conv_dbn_classification.py

@@ -72,11 +72,8 @@
 criterion = nn.CrossEntropyLoss()
 
 # Creating the optimzers
-optimizer = [
-    optim.Adam(model.models[0].parameters(), lr=0.0001),
-    optim.Adam(model.models[1].parameters(), lr=0.0001),
-    optim.Adam(fc.parameters(), lr=0.001),
-]
+optimizer = [optim.Adam(m.parameters(), lr=0.0001) for m in model.models]
+optimizer.append(optim.Adam(fc.parameters(), lr=0.001))
 
 # Creating training and validation batches
 train_batch = DataLoader(train, batch_size=batch_size, shuffle=True, num_workers=0)

examples/applications/deep/dbn_classification.py

@@ -28,11 +28,11 @@
 
 # Creating a DBN
 model = DBN(
-    model="bernoulli",
+    model=("gaussian", "sigmoid"),
     n_visible=784,
     n_hidden=(256, 256),
     steps=(1, 1),
-    learning_rate=(0.1, 0.1),
+    learning_rate=(0.0001, 0.001),
     momentum=(0, 0),
     decay=(0, 0),
     temperature=(1, 1),
@@ -59,12 +59,12 @@
 criterion = nn.CrossEntropyLoss()
 
 # Creating the optimzers
-optimizer = [optim.Adam(m.parameters(), lr=0.001) for m in model.models]
+optimizer = [optim.Adam(m.parameters(), lr=0.0001) for m in model.models]
 optimizer.append(optim.Adam(fc.parameters(), lr=0.001))
 
 # Creating training and validation batches
-train_batch = DataLoader(train, batch_size=batch_size, shuffle=False, num_workers=1)
-val_batch = DataLoader(test, batch_size=10000, shuffle=False, num_workers=1)
+train_batch = DataLoader(train, batch_size=batch_size, shuffle=True, num_workers=0)
+val_batch = DataLoader(test, batch_size=10000, shuffle=False, num_workers=0)
 
 # For amount of fine-tuning epochs
 for e in range(fine_tune_epochs):

learnergy/models/deep/conv_dbn.py

@@ -297,21 +297,22 @@ def fit(
                     dataset.targets.numpy(),
                     dataset.transform,
             )
+            d = Dataset(samples, targets, transform)
         except:
-            samples, targets, transform = (
-                    dataset.data,
-                    dataset.targets,
-                    dataset.transform,
-            )
-
-        d = Dataset(samples, targets, transform)
+            try:
+                samples, targets, transform = (
+                        dataset.data,
+                        dataset.targets,
+                        dataset.transform,
+                )
+                d = Dataset(samples, targets, transform)
+            except:
+                d = dataset
         batches = DataLoader(d, batch_size=batch_size, shuffle=True)
 
         for i, model in enumerate(self.models):
             logger.info("Fitting layer %d/%d ...", i + 1, self.n_layers)
 
-            #d = Dataset(samples, targets, transform)
-
             if i ==0:
                 model_mse = model.fit(d, batch_size, epochs[i])
                 mse.append(model_mse)

learnergy/models/deep/dbn.py

@@ -10,10 +10,12 @@
 import learnergy.utils.exception as e
 from learnergy.core import Dataset, Model
 from learnergy.models.bernoulli import RBM, DropoutRBM, EDropoutRBM
-from learnergy.models.extra import SigmoidRBM
+from learnergy.models.extra import SigmoidRBM, SigmoidRBM4deep
 from learnergy.models.gaussian import (
     GaussianRBM,
+    GaussianRBM4deep,
     GaussianReluRBM,
+    GaussianReluRBM4deep,
     GaussianSeluRBM,
     VarianceGaussianRBM,
 )
@@ -26,9 +28,12 @@
     "dropout": DropoutRBM,
     "e_dropout": EDropoutRBM,
     "gaussian": GaussianRBM,
+    "gaussian4deep": GaussianRBM4deep,
     "gaussian_relu": GaussianReluRBM,
+    "gaussian_relu4deep": GaussianReluRBM4deep,
     "gaussian_selu": GaussianSeluRBM,
     "sigmoid": SigmoidRBM,
+    "sigmoid4deep": SigmoidRBM4deep,
     "variance_gaussian": VarianceGaussianRBM,
 }
 
@@ -44,7 +49,7 @@ class DBN(Model):
 
     def __init__(
         self,
-        model: Optional[str] = "bernoulli",
+        model: Optional[Tuple[str, ...]] = ("gaussian", ),
         n_visible: Optional[int] = 128,
         n_hidden: Optional[Tuple[int, ...]] = (128,),
         steps: Optional[Tuple[int, ...]] = (1,),
@@ -53,6 +58,8 @@ def __init__(
         decay: Optional[Tuple[float, ...]] = (0.0,),
         temperature: Optional[Tuple[float, ...]] = (1.0,),
         use_gpu: Optional[bool] = False,
+        normalize: Optional[bool] = True,
+        input_normalize: Optional[bool] = True,
     ):
         """Initialization method.
 
@@ -84,27 +91,51 @@ def __init__(
         self.T = temperature
 
         self.models = []
+        model = list(model)
+        if len(model) < self.n_layers:
+            logger.info("\n\n> Incomplete number of RBMs, adding SigmoidRBMs to fill the stack!! <\n")
+            for i in range(len(model)-1, self.n_layers):
+                model.append("sigmoid4deep")
         for i in range(self.n_layers):
             if i == 0:
                 n_input = self.n_visible
             else:
                 # Gathers the number of input units as previous number of hidden units
                 n_input = self.n_hidden[i - 1]
 
-                # After creating the first layer, we need to change the model's type to sigmoid
-                model = "sigmoid"
-
-            m = MODELS[model](
-                n_input,
-                self.n_hidden[i],
-                self.steps[i],
-                self.lr[i],
-                self.momentum[i],
-                self.decay[i],
-                self.T[i],
-                use_gpu,
-            )
+                if model[i] == "sigmoid":
+                    model[i] = "sigmoid4deep"                    
+                elif model[i] == "gaussian":
+                    model[i] = "gaussian4deep"
+                elif model[i] == "gaussian_relu":
+                    model[i] = "gaussian_relu4deep"
+            try:
+                m = MODELS[model[i]](
+                        n_input,
+                        self.n_hidden[i],
+                        self.steps[i],
+                        self.lr[i],
+                        self.momentum[i],
+                        self.decay[i],
+                        self.T[i],
+                        use_gpu,
+                        normalize,
+                        input_normalize,
+                )
+
+            except:
+                m = MODELS[model[i]](
+                        n_input,
+                        self.n_hidden[i],
+                        self.steps[i],
+                        self.lr[i],
+                        self.momentum[i],
+                        self.decay[i],
+                        self.T[i],
+                        use_gpu,
+                )
             self.models.append(m)
+        model = tuple(model)
 
         if self.device == "cuda":
             self.cuda()
@@ -246,39 +277,65 @@ def fit(
 
         mse, pl = [], []
 
-        samples, targets, transform = (
-            dataset.data.numpy(),
-            dataset.targets.numpy(),
-            dataset.transform,
-        )
+        try: 
+            samples, targets, transform = (
+                    dataset.data.numpy(),
+                    dataset.targets.numpy(),
+                    dataset.transform,
+            )
+            d = Dataset(samples, targets, transform)
+        except:
+            try:
+                samples, targets, transform = (
+                        dataset.data,
+                        dataset.targets,
+                        dataset.transform,
+                )
+                d = Dataset(samples, targets, transform)
+            except:
+                d = dataset
+
+        batches = DataLoader(d, batch_size=batch_size, shuffle=True)
 
         for i, model in enumerate(self.models):
             logger.info("Fitting layer %d/%d ...", i + 1, self.n_layers)
-
-            d = Dataset(samples, targets, transform)
-
-            model_mse, model_pl = model.fit(d, batch_size, epochs[i])
-            mse.append(model_mse)
-            pl.append(model_pl)
-
-            if d.transform:
-                samples = d.transform(d.data)
-            else:
-                samples = d.data
-
-            if self.device == "cuda":
-                samples = samples.cuda()
-
-            samples = samples.reshape(len(dataset), model.n_visible)
-            targets = d.targets
-            transform = None
-
-            # Performs a forward pass over the samples to get their probabilities
-            samples, _ = model.hidden_sampling(samples)
-
-            if self.device == "cuda":
-                samples = samples.cpu()
-            samples = samples.detach()
+
+            if i ==0:
+                model_mse, model_pl = model.fit(d, batch_size, epochs[i])
+                mse.append(model_mse)
+                pl.append(model_pl)
+            else:                
+                # creating the training phase for deeper models
+                for ep in range(epochs[i]):
+                    logger.info("Epoch %d/%d", ep + 1, epochs[i])
+                    model_mse = 0
+                    pl_ = 0
+                    for step, (samples, y) in enumerate(batches):
+
+                        samples = samples.reshape(len(samples), self.n_visible)
+
+                        if self.device == "cuda":
+                            samples = samples.cuda()
+
+
+                        for ii in range(i):
+                            samples, _ = self.models[ii].hidden_sampling(samples)
+
+                        # Creating the dataset to ''mini-fit'' the i-th model                        
+                        ds = Dataset(samples, y, None, show_log=False)
+                        # Fiting the model with the batch
+                        mse_, plh = model.fit(ds, samples.size(0), 1)
+                        model_mse += mse_
+                        pl_ += plh
+
+                    model_mse/=len(batches)
+                    pl_/=len(batches)
+
+                    #logger.info("MSE: %f", model_mse)
+                    logger.info("MSE: %f | log-PL: %f", model_mse, pl_)
+                mse.append(model_mse)
+                pl.append(pl_)
+
 
         return mse, pl
 

learnergy/models/extra/__init__.py

@@ -1,4 +1,4 @@
 """A package contaning additional RBM-based models (networks) for all common learnergy modules.
 """
 
-from learnergy.models.extra.sigmoid_rbm import SigmoidRBM
+from learnergy.models.extra.sigmoid_rbm import SigmoidRBM, SigmoidRBM4deep