How to shuffle data at each epoch using API in TensorFlow 2.0? - python-3.x

I am getting my hands dirty using TensorFlow 2.0 to train my model. The new iteration feature in API is pretty awesome. However, when I was executing the following codes, I found that, unlike the iteration features in, it did not shuffle data automatically at each epoch. How do I achieve that using TF2.0?
import numpy as np
import tensorflow as tf
def sample_data():
data = sample_data()
# Subsample the data
mask = range(int(data.shape[0]*0.8), data.shape[0])
data_val = data[mask]
mask = range(int(data.shape[0]*0.8))
data_train = data[mask]
train_dset =\
val_dset =\
loss_metric = tf.keras.metrics.Mean(name='train_loss')
optimizer = tf.keras.optimizers.Adam(0.001)
def train_step(inputs):
for epoch in range(NUM_EPOCHS):
# Reset the metrics
for inputs in train_dset:

The batch needs to be reshuffled:
train_dset =\
train_dset = train_dset.shuffle(buffer_size=buffer_size)


How to predict Label of an email using a trained NB Classifier in sklearn?

I have created a Gaussian Naive Bayes classifier on a email (spam/not spam) dataset and was able to run it successfully. I vectorized the data, divided in it train and test sets and then calculated the accuracy, all the features that are present in the sklearn-Gaussian Naive Bayes classifier.
Now I want to be able to use this classifier to predict "labels" for new emails - whether they are by spam or not.
For example say I have an email. I want to feed it to my classifier and get the prediction as to whether it is a spam or not. How can I achieve this? Please Help.
Code for classifier file.
import sys
from time import time
import logging
# Display progress logs on stdout
logging.basicConfig(level = logging.DEBUG, format = '%(asctime)s %(message)s')
from vectorize_split_dataset import preprocess
### features_train and features_test are the features
for the training and testing datasets, respectively### labels_train and labels_test are the corresponding item labels
features_train, features_test, labels_train, labels_test = preprocess()
from sklearn.naive_bayes import GaussianNB
clf = GaussianNB()
t0 = time(), labels_train)
pred = clf.predict(features_test)
print("training time:", round(time() - t0, 3), "s")
print(clf.score(features_test, labels_test))
## Printing Metrics
for Training and Testing
print("No. of Testing Features:" + str(len(features_test)))
print("No. of Testing Features Label:" + str(len(labels_test)))
print("No. of Training Features:" + str(len(features_train)))
print("No. of Training Features Label:" + str(len(labels_train)))
print("No. of Predicted Features:" + str(len(pred)))
## Calculating Classifier Performance
from sklearn.metrics import classification_report
y_true = labels_test
y_pred = pred
labels = ['0', '1']
target_names = ['class 0', 'class 1']
print(classification_report(y_true, y_pred, target_names = target_names, labels = labels))
# How to predict label of a new text
new_text = "You won a lottery at UK lottery commission. Reply to claim it"
Code for Vectorization
import os
import pickle
import numpy
path = os.path.dirname(os.path.abspath(__file__))
### The words(features) and label_data(labels), already largely processed.###These files should have been created beforehand
feature_data_file = path + "./createdDataset/dataSet.pkl"
label_data_file = path + "./createdDataset/dataLabel.pkl"
feature_data = pickle.load(open(feature_data_file, "rb"))
label_data = pickle.load(open(label_data_file, "rb"))
### test_size is the percentage of events assigned to the test set(the### remainder go into training)### feature matrices changed to dense representations
for compatibility with### classifier functions in versions 0.15.2 and earlier
from sklearn import cross_validation
features_train, features_test, labels_train, labels_test = cross_validation.train_test_split(feature_data, label_data, test_size = 0.1, random_state = 42)
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(sublinear_tf = True, max_df = 0.5, stop_words = 'english')
features_train = vectorizer.fit_transform(features_train)
features_test = vectorizer.transform(features_test)#.toarray()
## feature selection to reduce dimensionality
from sklearn.feature_selection import SelectPercentile, f_classif
selector = SelectPercentile(f_classif, percentile = 5), labels_train)
features_train_transformed_reduced = selector.transform(features_train).toarray()
features_test_transformed_reduced = selector.transform(features_test).toarray()
features_train = features_train_transformed_reduced
features_test = features_test_transformed_reduced
def preprocess():
return features_train, features_test, labels_train, labels_test
Code for dataset generation
import os
import pickle
import re
import sys
# sys.path.append("../tools/")
Starter code to process the texts of accuate and inaccurate category to extract
the features and get the documents ready for classification.
The list of all the texts from accurate category are in the accurate_files list
likewise for texts of inaccurate category are in (inaccurate_files)
The data is stored in lists and packed away in pickle files at the end.
accurate_files = open("./rawDatasetLocation/accurateFiles.txt", "r")
inaccurate_files = open("./rawDatasetLocation/inaccurateFiles.txt", "r")
label_data = []
feature_data = []
### temp_counter is a way to speed up the development--there are### thousands of lines of accurate and inaccurate text, so running over all of them### can take a long time### temp_counter helps you only look at the first 200 lines in the list so you### can iterate your modifications quicker
temp_counter = 0
for name, from_text in [("accurate", accurate_files), ("inaccurate", inaccurate_files)]:
for path in from_text: ###only look at first 200 texts when developing### once everything is working, remove this line to run over full dataset
temp_counter = 1
if temp_counter < 200:
path = os.path.join('..', path[: -1])
text = open(path, "r")
line = text.readline()
while line: ###use a
function parseOutText to extract the text from the opened text# stem_text = parseOutText(text)
stem_text = text.readline().strip()
print(stem_text)### use str.replace() to remove any instances of the words# stem_text = stem_text.replace("germani", "")### append the text to feature_data
feature_data.append(stem_text)### append a 0 to label_data
if text is from Sara, and 1
if text is from Chris
if (name == "accurate"):
elif(name == "inaccurate"):
line = text.readline()
print("texts processed")
pickle.dump(feature_data, open("./createdDataset/dataSet.pkl", "wb"))
pickle.dump(label_data, open("./createdDataset/dataLabel.pkl", "wb"))
Also I want to know whether i can incrementally train the classifier meaning thereby that retrain a created model with newer data for refining the model over time?
I would be really glad if someone can help me out with this. I am really stuck at this point.
You are already using your model to predict labels of emails in your test set. This is what pred = clf.predict(features_test) does. If you want to see these labels, do print pred.
But perhaps you what to know how you can predict labels for emails that you discover in the future and that are not currently in your test set? If so, you can think of your new email(s) as a new test set. As with your previous test set, you will need to run several key processing steps on the data:
1) The first thing you need to do is to generate features for your new email data. The feature generation step is not included in your code above, but will need to occur.
2) You are using a Tfidf vectorizer, which converts a collection of documents to a matrix of Tfidf features based upon term frequency and inverse document frequency. You need to put your new email test feature data through the vectorizer that you fit on your training data.
3) Then your new email test feature data will need to go through dimensionality reduction using the same selector that you fit on your training data.
4) Finally, run predict on your new test data. Use print pred if you want to view the new label(s).
To respond to your final question about iteratively re-training your model, yes you definitely can do this. It's just a matter of selecting a frequency, producing a script that expands your data set with incoming data, then re-running all steps from there, from pre-processing to Tfidf vectorization, to dimensionality reduction, to fitting, and prediction.

How to create simple random array using Numba xoroshiro128p

I need to get simple example for creating random array using Numba xoroshiro128p within a JIT function. For example final array shell by size(2,4). Link for numba doc here
Pseudo code:
minimum = -2
maximum = 2
out_array = random(minimum, maximum, shape(2,4))
[[ 1.87569628 2.85881711 3.6009965 1.49224129]
[-3.27321953 1.59090995 -4.66912864 -3.43071647]]
Is possible to perform array creation with cuda faster then with numpy? For example:
minimum_bound = -1
maximum_bound = 1
vectors_number = 12000000
variable_number = 6
def random_matrix(vectors_number, variable_number):
population_generator = np.random.uniform(minimum_bound,
maximum_bound, (vectors_number, variable_number))
return population_generator
population_array = random_matrix(vectors_number, variable_number)
With 1200000 vectors to create i get fast the same speed as performing this on cuda.
The example in the documentation can be trivially modified to do what you want
from numba import cuda
from numba.cuda.random import create_xoroshiro128p_states, xoroshiro128p_uniform_float32
import numpy as np
def rand_array(rng_states, out):
thread_id = cuda.grid(1)
x = xoroshiro128p_uniform_float32(rng_states, thread_id)
out[thread_id] = x
threads_per_block = 4
blocks = 2
rng_states = create_xoroshiro128p_states(threads_per_block * blocks, seed=1)
out = np.zeros(threads_per_block * blocks, dtype=np.float32)
rand_array[blocks, threads_per_block](rng_states, out)

Memory Leak Keras TensorFlow1.8.0

I need help to minimize the memory leak suspected code .
I am using Keras latest with tensorflow 1.8.0 and python 3.6
When the program start its gradually growing to giganytes..!! need help here.
I am using VGG16 net for categorization to images. I couldnt localize the problem which causes the memory leaks.
Is it tensorflow bug or python suffering from such jobs
code is:
class_labels = ['cc','','cc','xx']
image = load_img(img_path, target_size=target_size)
image_arr = img_to_array(image) # convert from PIL Image to NumPy array
image_arr /= 255
image_arr = np.expand_dims(image_arr, axis=0)
model = applications.VGG16(include_top=False, weights='imagenet')
bottleneck_features = model.predict(image_arr)
model = create_top_model("softmax", bottleneck_features.shape[1:])
numpy_horizontal_concat = cv2.imread(img_path)
path ="/home/dataset/test"
listOfFiles = os.listdir(path)
pattern = "*.jpg"
model = applications.VGG16(include_top=False, weights='imagenet')
for entry in listOfFiles:
if fnmatch.fnmatch(entry, pattern):
image = load_img(path+"/"+ entry, target_size=target_size)
start_time = time.time()
image_arr = img_to_array(image) # convert from PIL Image to NumPy array
image_arr /= 255
image_arr = np.expand_dims(image_arr, axis=0)
bottleneck_features = model.predict(image_arr)
model2 = create_top_model("softmax", bottleneck_features.shape[1:])
predicted = model2.predict(bottleneck_features)
decoded_predictions = dict(zip(class_labels, predicted[0]))
decoded_predictions = sorted(decoded_predictions.items(), key=operator.itemgetter(1), reverse=True)
elapsed_time = time.time() - start_time
count = 1
for key, value in decoded_predictions[:5]:
print("{}. {}: {:8f}%".format(count, key, value * 100))
print("time: " , time.strftime("%H:%M:%S", time.gmtime(elapsed_time)) , " - " , elapsed_time)
count += 1
#OPENCV concat test
#numpy_horizontal_concat = np.concatenate((mat_image,numpy_horizontal_concat), axis=0)
hide_img = True
Inside your for loop you build a new model with loaded weights. This model is build inside your tensorflow session, which you don't reset. So you session is build up with many models without deleting a single one.
There are 2 possible solutions:
Try to optimize your code that you only have to load your model once. That way your code will get also much more faster
Reset your session:
I strongly recommend to use the first solution but if this isn't possible:
from keras import backend as K

Submit looping calculation to dask and get back the result

My co-worker and I have been setting up, configuring, and testing Dask for a week or so now, and everything is working great (can't speak highly enough about how easy, straightforward, and powerful it is), but now we are trying to leverage it for more than just testing and are running into an issue. We believe it's a fairly simple one related to syntax and an understanding gap. Any help to get it running is greatly appreciated. Any support in evolving our understanding of more optimal paths is also greatly appreciated.
We got fairly close with these two posts:
Dask: How would I parallelize my code with dask delayed?
Unpacking result of delayed function
High level flow:
Open data in pandas & clean it (we plan on moving this to a pipeline)
From there, convert the cleaned data set for regression into a dask data frame
Set the x & y variables and create all unique x combination sets
Create all unique formulas (y ~ x1 + x2 +0)
Run each individual formula set with the data through a linear lasso lars model to get the AIC for each formula for ranking
Current Issue:
Run each individual formula set (~1700 formulas) with the data (1 single data set which doesn’t vary with each run) on the dask cluster and get the results back
Optimize the calculation & return the final data
# In[]
# Imports:
import logging as log
import datetime as dat
from itertools import combinations
import numpy as np
import pandas as pd
from patsy import dmatrices
import sklearn as sk
from sklearn.linear_model import LogisticRegression, SGDClassifier, LinearRegression
import dask as dask
import dask.dataframe as dk
from dask.distributed import Client
# In[]
# logging, set the dask client, open & clean the data, pass into a dask dataframe
format='%(asctime)s %(message)s',
datefmt="%m-%d %H:%M:%S"
c = Client('ip:port')
ST =
data_pd = pd.read_csv('some.txt', sep="\t")
#fill some na/clean up the data a bit
data_pd['V9'] = data_pd.V9.fillna("Declined")
data_pd['y'] = data_pd.y.fillna(0)
data_pd['x1'] = data_pd.x1.fillna(0)
#output the clean data and re-import into dask, we could alse use from_pandas to get to dask dataframes
data = dk.read_csv(r'C:\path\*.csv', sep=",")
# set x & y variables - the below is truncated
y_var = "y"
x_var = ['x1',
#list of all variables
all_var = list(y_var) + x_var
#all unique combinations
x_var_combos = [combos for combos in combinations(x_var,2)]
#add single variables for testing as well
for i in x_var:
# create formulas from our y, x variables
def formula(y_var, combo):
combo_len = len(combo)
if combo_len == 2:
formula = y_var +"~"+combo[0] +"+"+ combo[1]+"+0"
formula = y_var +"~"+combo[0]+"+0"
return formula
def model_aic(dt, formula):
k = 2
y_df, x_df = dmatrices(formula, dt, return_type = 'dataframe')
y_df = np.ravel(y_df)'dmatrices successful')
LL_model = sk.linear_model.LassoLarsIC(max_iter = 100)
AIC_Value = min(, y_df).criterion_) + ( (2*(k**2)+2*(k)) / (len(x_df)-k-1) )'AIC_Value: %s', AIC_Value)
oup = [formula ,AIC_Value, len(dt)-AIC_Value]
return oup
# ----------------- here's where we're stuck ---------------------
# ----------------- we think this is correct ----------------------
# ----------------- create a list of all formula to execute -------
# In[]
out = []
for i in x_var_combos:
var = model_aic(data, formula(y_var, i))
# ----------------- but we're stuck figuring out how to -----------
# ------------------make it compute & return the result -----------
ans = c.compute(*out)
ans2 = c.compute(out[1])
print (ans2)

tensorboard with numpy array

Can someone give a example on how to use tensorboard visualize numpy array value?
There is a related question here, I don't really get it.
Tensorboard logging non-tensor (numpy) information (AUC)
For example,
If I have
for i in range(100):
foo = np.random.rand(3,2)
How can I keep tracking the distribution of foo using tensorboard for 100 iterations? Can someone give a code example?
For simple values (scalar), you can use this recipe
summary_writer = tf.train.SummaryWriter(FLAGS.logdir)
summary = tf.Summary()
summary.value.add(tag=tagname, simple_value=value)
summary_writer.add_summary(summary, global_step)
As far as using array, perhaps you can add 6 values in a sequence, ie
for value in foo:
summary.value.add(tag=tagname, simple_value=value)
Another (simplest) way is just using placeholders. First, you can make a placeholder for your numpy array shape.
# Some place holders for summary
summary_reward = tf.placeholder(tf.float32, shape=(), name="reward")
tf.summary.scalar("reward", summary_reward)
Then, just call the merged summary with the feed_dict.
# Summary
summ = tf.summary.merge_all()
s =, feed_dict={summary_reward: reward})
writer.add_summary(s, i)
if you install this package via pip install tensorboard-pytorch it becomes as straightforward as it can get:
import numpy as np
from tensorboardX import SummaryWriter
writer = SummaryWriter()
for i in range(50):
writer.add_histogram("moving_gauss", np.random.normal(i, i, 1000), i, bins="auto")
Will generate the corresponding histogram data in the runs directory:
Found a way to work around, create a variable and assign the value of numpy array to the variable, use tensorboard to track the variable
mysummary_writer = tf.train.SummaryWriter("./tmp/test/")
a = tf.Variable(tf.zeros([3,2]), name="a")
sum1 = tf.histogram_summary("nparray1", a)
summary_op = tf.merge_all_summaries()
sess = tf.Session()
for ii in range(10):
foo = np.random.rand(3, 2)
assign_op = a.assign(foo)
summary, _ =[summary_op, assign_op])
mysummary_writer.add_summary(tf.Summary.FromString(summary), global_step=ii)
sess = tf.Session()
writer = tf.summary.FileWriter('tensorboard_test')
var = tf.Variable(0.0,trainable=False,name='loss')
summary_op = tf.summary.scalar('scalar1',var)
for value in array:
summary =
It works, but slow.
You could define a function like this (taken from gyglim's gist):
def add_histogram(writer, tag, values, step, bins=1000):
Logs the histogram of a list/vector of values.
# Create histogram using numpy
counts, bin_edges = np.histogram(values, bins=bins)
# Fill fields of histogram proto
hist = tf.HistogramProto()
hist.min = float(np.min(values))
hist.max = float(np.max(values))
hist.num = int(
hist.sum = float(np.sum(values))
hist.sum_squares = float(np.sum(values ** 2))
# Requires equal number as bins, where the first goes from -DBL_MAX to bin_edges[1]
# See
# Therefore we drop the start of the first bin
bin_edges = bin_edges[1:]
# Add bin edges and counts
for edge in bin_edges:
for c in counts:
# Create and write Summary
summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])
writer.add_summary(summary, step)
And then add to the summary writer like this:
add_histogram(summary_writer, "Histogram_Name", your_numpy_array, step)