TensorFlow 2.x and Keras must keep snippets

# Source: https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/quickstart/beginner.ipynb

import tensorflow as tf
mnist = tf.keras.datasets.mnist

(x_train, y_train),(x_test, y_test) = mnist.load_data()
# Normalize the inputs
x_train, x_test = x_train / 255., x_test / 255.

model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(input_shape=(28, 28)),
  tf.keras.layers.Dense(128, activation='relu'),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10, activation='softmax')
])

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

model.fit(x_train, y_train, epochs=5)
model.evaluate(x_test, y_test)

# Source: https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/quickstart/advanced.ipynb

import tensorflow as tf
mnist = tf.keras.datasets.mnist

(x_train, y_train),(x_test, y_test) = mnist.load_data()
# Normalize the inputs
x_train, x_test = x_train / 255., x_test / 255.

class MyModel(tf.keras.Model):
  def __init__(self):
    super(MyModel, self).__init__()
    self.conv1 = Conv2D(32, 3, activation='relu')
    self.flatten = Flatten()
    self.d1 = Dense(128, activation='relu')
    self.d2 = Dense(10, activation='softmax')

  def call(self, x):
    x = self.conv1(x)
    x = self.flatten(x)
    x = self.d1(x)
    return self.d2(x)
model = MyModel()

loss_object = tf.keras.losses.SparseCategoricalCrossentropy()
optimizer = tf.keras.optimizers.Adam()

with tf.GradientTape() as tape:
  logits = model(images)
  loss_value = loss_object(logits, labels)
grads = tape.gradient(loss_value, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))

@tf.function
def increment_even(x):
  if x % 2 == 0:
    x += 1

  return x

from tensorflow.keras.preprocessing.image import ImageDataGenerator

train_datagen = ImageDataGenerator(
    # Normalize the input
    rescale=1/255,
    # Augmentation specs
    rotation_range=40,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True
)
test_datagen = ImageDataGenerator(
    # Normalize the input
    rescale=1/255
)

train_generator = train_datagen.flow_from_directory(
    some_path,  # Path to your data.
    target_size=(150, 150),
    batch_size=batch_size,  # Your training batch size
    class_mode='categorical'  # You can also `binary` for binary classication tasks.
)

test_generator = test_datagen.flow_from_directory(
    # Use the same parameters as in train_generator.
)

# Finally we train the model
model.fit_generator(
    train_generator,
    steps_per_epoch=len(train_generator),
    epochs=50  # You can also add validation generators here.
)

# Evaluate the model
loss, accuracy = model.evaluate_generator(
    test_generator,
    steps=len(test_generator)
)

import tensorflow as tf
from tensorflow.keras import datasets
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing.sequence import pad_sequences

# Load data
vocab_size = 10000
# If you are loading your own corpus you will need to tokenize it first using `tensorflow.keras.preprocessing.text.Tokenizer.fit_on_texts`
(train_x, train_y), (test_x, test_y) = datasets.reuters.load_data(num_words=vocab_size)

# Pad the sequences so that they would have the same length
max_length = 200
train_x = pad_sequences(train_x, maxlen=max_length)
test_x = pad_sequences(test_x, maxlen=max_length)

# One hot encode y
number_of_classes = len(np.unique(train_y))
train_y = to_categorical(train_y, number_of_classes)
test_y = to_categorical(test_y, number_of_classes)

# Create the model
model = Sequential()
model.add(layers.Embedding(vocab_size, 1024, input_length=max_length))
model.add(layers.GlobalAveragePooling1D())
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(46, activation='softmax'))
model.summary()

# Training...
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(x=train_x, y=train_y, validation_split=0.2, shuffle=True, batch_size=512, epochs=20)

# Evaluate the model 
loss, accuracy = model.evaluate(x=test_x, y=test_y)

from tensorflow.keras.preprocessing.text import Tokenizer

# Create the tokenizer
tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)

# Summary
print(tokenizer.word_counts)
print(tokenizer.document_count)
print(tokenizer.word_index)
print(tokenizer.word_docs)

from pathlib import Path

vocab_size = 10000
embedding_size = 100

# Build your model
model = Sequential()
embedding_layer = layers.Embedding(vocab_size, embedding_size, input_length=max_length)
model.add(embedding_layer)
# Add more layers...

# Download pre-trained word embeddings
# I will use the smallest available pre-trained word vectors from [GloVe](https://nlp.stanford.edu/projects/glove/) 
# which should be more than enough. That was glove.6B.zip which consists of 6B tokens, 400K vocab, uncased, 50d, 100d, 200d, & 300d vectors. 
# It is an 822 MB download.
embedding_path = Path("../embedding")
embedding_path.mkdir(parents=True, exist_ok=True)

!wget -O {embedding_path}/glove.6B.zip http://nlp.stanford.edu/data/glove.6B.zip
!unzip {embedding_path}/glove.6B.zip -d {embedding_path}

# Load embeddings
embedding_index = {}
with open(embedding_path/"glove.6B.100d.txt", "r") as reader:
    line = reader.readline()
    while line != '':  # The EOF char is an empty string
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[1:], np.float32)
        embedding_index[word] = coefs

        line = reader.readline()

# Set embedding_layer weights
embedding_matrix = np.zeros((vocab_size, embedding_size))
for word, index in word_index.items():
    if index >= vocab_size:
        continue

    coefs = embedding_index.get(word)
    if coefs is not None:
        embedding_matrix[index] = coefs

embedding_layer.set_weights([embedding_matrix])
embedding_layer.trainable = False

# TODO :: Compile model here for trainable change to take effect
# model.compile()

import matplotlib.pyplot as plt
%matplotlib inline

# Train model.
history = model.fit(x=train_x, y=train_y, validation_split=0.2, shuffle=True, batch_size=128, epochs=10)
print(history.history.keys())

# Plot training progress.
fig = plt.figure(figsize=(20, 10))

plt.subplot(1, 2, 1)
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('Model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')

plt.subplot(1, 2, 2)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')

plt.show()

from tensorflow import keras

callbacks = [
    keras.callbacks.TensorBoard(log_dir='../logs/dogs_vs_cats_v2')
]

history = model.fit_generator(
    ...
    callbacks=callbacks,
    ...
)

# Then check progress in TensorBoard by running `tensorboard --logdir=logs/dogs_vs_cats_v2` in terminal

from tensorflow import keras

initial_learning_rate = 0.1
learning_rate_schedule = keras.optimizers.schedules.ExponentialDecay(
    initial_learning_rate,
    decay_steps=100000,
    decay_rate=0.96,
    staircase=True
)

optimizer = keras.optimizers.RMSprop(learning_rate=learning_rate_schedule)

from tensorflow import keras

reduce_lr = keras.callbacks.ReduceLROnPlateau(
    monitor='val_loss', 
    factor=0.1,  # Divide the learning rate by 10 when triggered
    patience=5,  # Trigger when val_loss has stopped improving for 5 epochs 
    min_lr=0.001
)

model.fit(X_train, Y_train, callbacks=[reduce_lr])

from tensorflow import keras

callbacks = [
    # Stop the training when there is no improvement in the validation loss for three consecutive epochs.
    keras.callbacks.EarlyStopping(
        monitor='val_loss', 
        patience=3
    ),
    # Save model after every epoch or save the best one so far only 
    keras.callbacks.ModelCheckpoint(
        file_path="_model.h5",
        monitor="val_loss",
        save_best_only=True
    ),
    # Check training progress in TensorBoard
    keras.callbacks.TensorBoard(
        log_dir="model_log_dir",
        histogram_freq=1,  # Records activation histograms every 1 epoch
        embedding_freq = 1  # Record emedding data every 1 epoch. If set to 0, embeddings won't be visualized.
    )
]

model.fit(X_train, Y_train, callbacks=callbacks)

model.save('path_to_my_model.h5')

# Load the model for inference
model = keras.models.load_model('path_to_my_model.h5')

model.trainable = False
# TODO :: Compile model here for trainable change to take effect
# model.compile()

# Make sure this model does not have trainable variables
model.trainable_variables

# Source: https://www.tensorflow.org/alpha/guide/checkpoints
import tensorflow as tf

opt = tf.keras.optimizers.Adam(0.1)
net = Net()

ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=net)
manager = tf.train.CheckpointManager(ckpt, 'tf_ckpts', max_to_keep=3)
ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
    print(f'Restored from {manager.latest_checkpoint}')
else:
    print('Initializing from scratch.')

for example in toy_dataset():
    loss = train_step(net, example, opt)
    ckpt.step.assign_add(1)
    if int(ckpt.step) % 10 == 0:
        save_path = manager.save()
        print(f'Saved checkpoint for step {int(ckpt.step)}: {save_path}')
        print(f'loss {loss.numpy():1.2f}')