We Keep Coding

Hugging Face Spaces failing to build using fast.ai because of Resampling on module PIL.Image

I keep getting the below error when I try to access my model on hugging face spaces. I am building my model in a Kaggle notebook, then downloading to a pkl file to my spaces repo and git pushing to HF spaces. Below is my ImageDataLoaders class that I am using, as I suspect the error is coming from here.
dls = ImageDataLoaders.from_folder(path, valid_pct = 0.2, item_tfms=Resize(460), batch_tfms=aug_transforms(size=224, min_scale=0.75))
Here is the error I am getting.
Traceback (most recent call last):
File "app.py", line 5, in <module>
learn = load_learner('new_model.pkl')
File "/home/user/.local/lib/python3.8/site-packages/fastai/learner.py", line 428, in load_learner
try: res = torch.load(fname, map_location=map_loc, pickle_module=pickle_module)
File "/home/user/.local/lib/python3.8/site-packages/torch/serialization.py", line 712, in load
return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
File "/home/user/.local/lib/python3.8/site-packages/torch/serialization.py", line 1049, in _load
result = unpickler.load()
File "/home/user/.local/lib/python3.8/site-packages/torch/serialization.py", line 1042, in find_class
return super().find_class(mod_name, name)
AttributeError: Custom classes or functions exported with your `Learner` not available in namespace.\Re-declare/import before loading:
Can't get attribute 'Resampling' on <module 'PIL.Image' from '/home/user/.local/lib/python3.8/site-packages/PIL/Image.py'>
Here is my fill app.py code.
from fastai.vision.all import *
import gradio as gr
import skimage
learn = load_learner('new_model.pkl')
categories = ('deer', 'elk', 'moose')
def classify_image(img):
img = PILImage.create(img)
pred, idx, probs = learn.predict(img)
return dict(zip(categories, map(float,probs)))
image = gr.inputs.Image(type='pil', shape=(192,192))
label = gr.outputs.Label()
intf = gr.Interface(fn=classify_image, inputs=image, outputs=label)
intf.launch(inline=False)

RuntimeError: Unknown qengine?The code is written correctly, but there is a problem, why and how to solve it?

import torch
import sounddevice as sd
import time
language = "ru"
model_id = "ru_v3"
sample_rate = 48000
speaker = "baya"
put_accent = True
put_yo = True
device = torch.device('cpu')
text = "Здраствуй Олег, рад знакомству"
model, _ = torch.hub.load(repo_or_dir='snakers4/silero-models',
model='silero_tts',
language=language,
speaker=model_id
)
model.to(device)
audio = model.apply_tts(text=text,
speaker=speaker,
sample_rate=sample_rate,
put_accent=put_accent,
put_yo=put_yo)
print(text)
sd.play(audio, sample_rate)
time.sleep(len(audio) / sample_rate)
sd.stop()
Error message:
Using cache found in C:\Users\User/.cache\torch\hub\snakers4_silero-models_master
Traceback (most recent call last):
File "D:/Voice_Assistent2/main.py", line 14, in <module>
model, _ = torch.hub.load(repo_or_dir='snakers4/silero-models',
File "D:\Voice_Assistent2\venv\lib\site-packages\torch\hub.py", line 399, in load
model = _load_local(repo_or_dir, model, *args, **kwargs)
File "D:\Voice_Assistent2\venv\lib\site-packages\torch\hub.py", line 428, in _load_local
model = entry(*args, **kwargs)
File "C:\Users\User/.cache\torch\hub\snakers4_silero-models_master\src\silero\silero.py", line 88, in silero_tts
model = imp.load_pickle("tts_models", "model")
File "D:\Voice_Assistent2\venv\lib\site-packages\torch\package\package_importer.py", line 249, in load_pickle
result = unpickler.load()
File "C:\Users\User\AppData\Local\Programs\Python\Python38\lib\pickle.py", line 1210, in load
dispatch[key[0]](self)
File "C:\Users\User\AppData\Local\Programs\Python\Python38\lib\pickle.py", line 1251, in load_binpersid
self.append(self.persistent_load(pid))
File "D:\Voice_Assistent2\venv\lib\site-packages\torch\package\package_importer.py", line 227, in persistent_load
loaded_reduces[reduce_id] = func(self, *args)
File "D:\Voice_Assistent2\venv\lib\site-packages\torch\jit\_script.py", line 344, in unpackage_script_module
cpp_module = torch._C._import_ir_module_from_package(
RuntimeError: Unknown qengine

GPT 2 - TypeError: Cannot cast array data from dtype('O') to dtype('int64') according to the rule 'safe'

I am working with gpt2, python 3.9 and tensorflow 2.5 and when connecting to flask (flask run in terminal) I get a following message:
TypeError: Cannot cast array data from dtype('O') to dtype('int64') according to the rule 'safe'
Here is the code in generator.py
#!/usr/bin/env python3
import fire
import json
import os
import numpy as np
import tensorflow.compat.v1 as tf
# import model, sample, encoder
from text_generator import model
from text_generator import sample
from text_generator import encoder
class AI:
def generate_text(self, input_text):
model_name = '117M_Trained'
seed = None,
nsamples = 1
batch_size = 1
length = 150
temperature = 1
top_k = 40
top_p = 1
models_dir = 'models'
self.response = ''
models_dir = os.path.expanduser(os.path.expandvars(models_dir))
if batch_size is None:
batch_size = 1
assert nsamples % batch_size == 0
enc = encoder.get_encoder(model_name, models_dir)
hparams = model.default_hparams()
cur_path = os.path.dirname(__file__) + '/models' + '/' + model_name
with open(cur_path + '/hparams.json') as f:
hparams.override_from_dict(json.load(f))
if length is None:
length = hparams.n_ctx // 2
elif length > hparams.n_ctx:
raise ValueError("Can't get samples longer than window size: %s" % hparams.n_ctx)
with tf.Session(graph=tf.Graph()) as sess:
context = tf.placeholder(tf.int32, [batch_size, None])
np.random.seed(seed)
tf.set_random_seed(seed)
output = sample.sample_sequence(
hparams=hparams, length=length,
context=context,
batch_size=batch_size,
temperature=temperature, top_k=top_k, top_p=top_p
)
saver = tf.train.Saver()
ckpt = tf.train.latest_checkpoint(cur_path)
saver.restore(sess, ckpt)
context_tokens = enc.encode(input_text)
generated = 0
for _ in range(nsamples // batch_size):
out = sess.run(output, feed_dict={
context: [context_tokens for _ in range(batch_size)]
})[:, len(context_tokens):]
for i in range(batch_size):
generated += 1
text = enc.decode(out[i])
self.response = text
return self.response
ai = AI()
text = ai.generate_text('How are you?')
print(text)
Any help is appreciated 🙏 ps I have also added below the entire traceback
* Serving Flask app 'text_generator' (lazy loading)
* Environment: development
* Debug mode: on
2021-09-14 19:58:08.687907: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
Traceback (most recent call last):
File "_mt19937.pyx", line 178, in numpy.random._mt19937.MT19937._legacy_seeding
TypeError: 'tuple' object cannot be interpreted as an integer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/dusandev/miniconda3/bin/flask", line 8, in <module>
sys.exit(main())
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/flask/cli.py", line 990, in main
cli.main(args=sys.argv[1:])
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/flask/cli.py", line 596, in main
return super().main(*args, **kwargs)
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/click/core.py", line 1062, in main
rv = self.invoke(ctx)
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/click/core.py", line 1668, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/click/core.py", line 763, in invoke
return __callback(*args, **kwargs)
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/click/decorators.py", line 84, in new_func
return ctx.invoke(f, obj, *args, **kwargs)
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/click/core.py", line 763, in invoke
return __callback(*args, **kwargs)
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/flask/cli.py", line 845, in run_command
app = DispatchingApp(info.load_app, use_eager_loading=eager_loading)
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/flask/cli.py", line 321, in __init__
self._load_unlocked()
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/flask/cli.py", line 346, in _load_unlocked
self._app = rv = self.loader()
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/flask/cli.py", line 402, in load_app
app = locate_app(self, import_name, name)
File "/Users/dusandev/miniconda3/lib/python3.9/site-packages/flask/cli.py", line 256, in locate_app
__import__(module_name)
File "/Users/dusandev/Desktop/AI/text_generator/__init__.py", line 2, in <module>
from .routes import generator
File "/Users/dusandev/Desktop/AI/text_generator/routes.py", line 2, in <module>
from .generator import ai
File "/Users/dusandev/Desktop/AI/text_generator/generator.py", line 74, in <module>
text = ai.generate_text('How are you?')
File "/Users/dusandev/Desktop/AI/text_generator/generator.py", line 46, in generate_text
np.random.seed(seed)
File "mtrand.pyx", line 244, in numpy.random.mtrand.RandomState.seed
File "_mt19937.pyx", line 166, in numpy.random._mt19937.MT19937._legacy_seeding
File "_mt19937.pyx", line 186, in numpy.random._mt19937.MT19937._legacy_seeding
TypeError: Cannot cast array data from dtype('O') to dtype('int64') according to the rule 'safe'

The problem is the line None, in your code. This is causing the tuple (None,) as the input to the np.random.seed(seed). It accepts integer, but you are sending the tuple.

tensorflow.python.framework.errors_impl.UnimplementedError: Cast string to int32 is not supported

I only wrote a simple model in model.py, and when I ran it, it gave the following error.
2021-02-08 22:20:11.872409: E tensorflow/core/common_runtime/executor.cc:641] Executor failed to create kernel. Unimplemented: Cast string to int32 is not supported
[[{{node embedding/Cast}}]]
Traceback (most recent call last):
File "C:\Users\xiaoc\Anaconda3\lib\runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "C:\Users\xiaoc\Anaconda3\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\xiaoc\AppData\Local\Google\Cloud SDK\trainer\task.py", line 55, in
train_model(args)
File "C:\Users\xiaoc\AppData\Local\Google\Cloud SDK\trainer\task.py", line 43, in train_model
validation_data=(eval_data, eval_labels))
File "C:\Users\xiaoc\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\training.py", line 780, in fit
steps_name='steps_per_epoch')
File "C:\Users\xiaoc\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\training_arrays.py", line 363, in model_iteration
batch_outs = f(ins_batch)
File "C:\Users\xiaoc\Anaconda3\lib\site-packages\tensorflow\python\keras\backend.py", line 3289, in call
self._make_callable(feed_arrays, feed_symbols, symbol_vals, session)
File "C:\Users\xiaoc\Anaconda3\lib\site-packages\tensorflow\python\keras\backend.py", line 3222, in _make_callable
callable_fn = session._make_callable_from_options(callable_opts)
File "C:\Users\xiaoc\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1489, in _make_callable_from_options
return BaseSession._Callable(self, callable_options)
File "C:\Users\xiaoc\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1446, in init
session._session, options_ptr)
tensorflow.python.framework.errors_impl.UnimplementedError: Cast string to int32 is not supported
[[{{node embedding/Cast}}]]
What is the problem? The requirement is to only make changes in model.py, not others. Thanks in advance!
Following are the three python files.
model.py
import tensorflow as tf
from tensorflow.keras.layers import Dense,Embedding,LSTM, Activation,Dropout
from tensorflow.keras import Model
def get_batch_size(): #size of training 8056 number of batches = 8056/128
return 128
def get_epochs():
return 50
def solution(input_layer):
max_len = 150
max_words = 200
# inputs = Input(name='inputs',shape=[max_len])
layer = Embedding(max_words,output_dim = 64, input_length=max_len)(input_layer)
# layer = LSTM(64,return_sequences=True)(input_layer)
layer = tf.expand_dims(layer, axis=-1)
layer = LSTM(64,return_sequences=True)(layer)
layer = Dense(256,)(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.5)(layer)
layer = Dense(5)(layer)
# layer = Activation('softmax')(layer)
model = Model(inputs=input_layer,outputs=layer)
model.compile(loss='sparse_categorical_crossentropy',optimizer=tf.keras.optimizers.Adam(),metrics=['accuracy'])
return model
data.py
import csv
import numpy as np
label_map = {
0: 'A',
1: 'B',
2: 'C',
3: 'D',
4: 'E',
}
label_map_inv = dict(map(reversed, label_map.items()))
def load_dataset(dataset_file):
data = []
labels = []
with open(dataset_file, "r", encoding="utf-8") as f:
data_reader = csv.reader(f, delimiter=",", quotechar='"')
next(data_reader)
for lbl, desc in data_reader:
data.append(desc)
labels.append(label_map_inv[lbl])
return np.array(data), np.array(labels)
task.py
import os
import argparse
import logging
import numpy as np
import tensorflow as tf
import tensorflow.keras
import trainer.data as data
import trainer.model as model
def train_model(params):
(train_data, train_labels) = data.load_dataset("data/train.csv")
(eval_data, eval_labels) = data.load_dataset("data/eval.csv")
input_layer = tf.keras.Input(shape=(), name='input_text', dtype=tf.string)
ml_model = model.solution(input_layer)
if ml_model is None:
print("No model found. You need to implement one in model.py")
else:
ml_model.fit(train_data, train_labels,
batch_size=model.get_batch_size(),
epochs=model.get_epochs(),
validation_data=(eval_data, eval_labels))
_ = ml_model.evaluate(eval_data, eval_labels, verbose=1)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
args = parser.parse_args()
tf_logger = logging.getLogger("tensorflow")
tf_logger.setLevel(logging.INFO)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = str(tf_logger.level // 10)
train_model(args)

Error loading .npz files in tensorflow dataset

I'm trying to create a data pipeline in tensorflow, but my data is in .npz files.
Following the documentation at https://www.tensorflow.org/guide/data#consuming_sets_of_files, for consuming sets of files, combined with using tf.py_function() for using numpy ops, I wrote a code, a subsection of which is:
def load_data(filename):
from preprocess import normalize
# sess = tf.compat.v1.Session()
# fln = sess.run(filename)
print ('I AM TRYING TO LOAD : ', filename)
mels = np.load(filename)['arr_0']
mels = normalize(mels)
return mels
I'm getting an error in np.load(filename), which goes like:
I AM TRYING TO LOAD : tf.Tensor(b'/media/prabhatk/Datasets/DCASE/features/augmentations=None features=mono fmax=22050 fmin=0 hop_length=1024 mel_htk=True n_fft=2048 n_mels=60 samplerate=44100/airport-lisbon-1000-40000-a.npz', shape=(), dtype=string)
2020-02-18 19:31:31.048435: W tensorflow/core/framework/op_kernel.cc:1610] Invalid argument: TypeError: expected str, bytes or os.PathLike object, not tensorflow.python.framework.ops.EagerTensor
Fixed this by using np.load(filename.numpy()), as stated by #jdehesa.
Now I run into some shape issues, although I've already reshaped the input in the load_data_wrapper() function.
Can anyone help me with this?
I am attaching the entire code and entire error message below.
CODE:
import os
from os import path
import sys
import argparse
sys.path.insert(1,'../')
from helpers import locations
from helpers import metadata
from helpers import read_settings as Settings
from models import sbcnn
import numpy as np
import pandas as pd
import librosa as lb
from sklearn.preprocessing import LabelEncoder
import tensorflow as tf
# from tf.data.experimental import AUTOTUNE
def load_data(filename):
from preprocess import normalize
# sess = tf.compat.v1.Session()
# fln = sess.run(filename)
print ('I AM TRYING TO LOAD : ', filename)
mels = np.load(filename)['arr_0']
# mels = tf.io.read_file(filename)['arr_0']
mels = normalize(mels)
return mels
def load_data_wrapper(filename):
# print ('INPUT TO LOAD DATA IS : ', filename)
[mels,] = tf.py_function(
load_data, [filename], [tf.float32]
)
mels.set_shape((1,60,431,1))
# return mels
return mels
def get_input_directory(input_dir, feature_settings):
base_dir = input_dir if (input_dir != '') else locations.FEATURE_DIR_BASE
assert path.exists(base_dir), 'Feature directory not found!!'
settings = Settings.load_settings(feature_settings)
feature_dir = Settings.settings_to_path(settings)
feature_dir = path.join(base_dir,feature_dir)
assert path.exists(feature_dir), 'Data not preprocessed according to given settings!!'
return feature_dir
def construct_dataset(X, y):
filepaths = tf.data.Dataset.from_tensor_slices(X)
with tf.device('/device:CPU:*'):
files = filepaths.map(load_data_wrapper)
filelabels = tf.data.Dataset.from_tensor_slices(y)
return tf.data.Dataset.zip((files, filelabels))
def parseArguments():
parser = argparse.ArgumentParser()
parser.add_argument('--training_metadata', type=str, default='', help='CSV file containing training file names and lables')
parser.add_argument('--validation_metadata', type=str, default='', help='CSV file containing validation file names and lables')
parser.add_argument('--input_dir', type=str, default='', help='Processed features directory')
parser.add_argument('--feature_settings', type=str, default='', help='Load data with the given settings')
args = parser.parse_args()
return args
def main():
arguments = parseArguments()
input_dir = get_input_directory(arguments.input_dir, arguments.feature_settings)
X_tr, y_tr = metadata.train(arguments.training_metadata, input_dir)
X_val, y_val = metadata.validation(arguments.validation_metadata, input_dir)
enc = LabelEncoder()
enc.fit(y_tr)
y_tr = enc.transform(y_tr)
y_val = enc.transform(y_val)
data_tr = construct_dataset(X_tr, y_tr)
data_val = construct_dataset(X_val, y_val)
model = sbcnn.build_model()
model.compile(loss='categorical_crossentropy',
optimizer=tf.keras.optimizers.SGD(),
metrics=['accuracy'])
model.summary()
model.fit(data_tr, epochs=10)
if __name__ == '__main__':
main()
ERROR:
2020-02-19 10:37:06.375134: W tensorflow/core/framework/op_kernel.cc:1622] OP_REQUIRES failed at conv_ops_fused_impl.h:693 : Invalid argument: input must be 4-dimensional[60,431]
2020-02-19 10:37:06.375184: W tensorflow/core/common_runtime/base_collective_executor.cc:216] BaseCollectiveExecutor::StartAbort Invalid argument: input must be 4-dimensional[60,431]
[[{{node sequential/conv2d/BiasAdd}}]]
1/Unknown - 0s 389ms/stepTraceback (most recent call last):
File "pipeline.py", line 112, in <module>
main()
File "pipeline.py", line 108, in main
model.fit(data_tr, epochs=10)
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py", line 728, in fit
use_multiprocessing=use_multiprocessing)
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 324, in fit
total_epochs=epochs)
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 123, in run_one_epoch
batch_outs = execution_function(iterator)
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 86, in execution_function
distributed_function(input_fn))
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py", line 457, in __call__
result = self._call(*args, **kwds)
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py", line 520, in _call
return self._stateless_fn(*args, **kwds)
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 1823, in __call__
return graph_function._filtered_call(args, kwargs) # pylint: disable=protected-access
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 1141, in _filtered_call
self.captured_inputs)
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 1224, in _call_flat
ctx, args, cancellation_manager=cancellation_manager)
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 511, in call
ctx=ctx)
File "/home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py", line 67, in quick_execute
six.raise_from(core._status_to_exception(e.code, message), None)
File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: input must be 4-dimensional[60,431]
[[node sequential/conv2d/BiasAdd (defined at /home/prabhatk/miniconda3/envs/DL/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py:1751) ]] [Op:__inference_distributed_function_801]
Function call stack:
distributed_function