I have selected the CIFAR 10 dataset using the torchvision library:
trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
download=True, transform=transforms.ToTensor())
Then I try to select the first item in the dataset, which as I understand implements the get_item method of the dataset class:
trainset[0]
and I get
File "env\lib\site-packages\torchvision\transforms\functional.py", line 129, in to_tensor
np.array(pic, mode_to_nptype.get(pic.mode, np.uint8), copy=True)
TypeError: __array__() takes 1 positional argument but 2 were given
Any ideas why I would get this error?
Python 3.7.9, torch==1.9.0, torchvision==0.10.0
I was hitting this error too:
def get_transformations():
return transforms.Compose([transforms.ToTensor()])
...
self.transforms = get_transformations()
...
# Load the image + augment
img = Image.open(img_path).convert("RGB")
img = self.transforms(img)
...
Original Traceback (most recent call last):
File "c:\2021-mcm-master\src\PyTorch-RCNN\ui-prediction\env\lib\site-packages\torch\utils\data\_utils\worker.py", line 287, in _worker_loop
data = fetcher.fetch(index)
File "c:\2021-mcm-master\src\PyTorch-RCNN\ui-prediction\env\lib\site-packages\torch\utils\data\_utils\fetch.py", line 44, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "c:\2021-mcm-master\src\PyTorch-RCNN\ui-prediction\env\lib\site-packages\torch\utils\data\_utils\fetch.py", line 44, in <listcomp>
data = [self.dataset[idx] for idx in possibly_batched_index]
File "c:\2021-mcm-master\src\PyTorch-RCNN\ui-prediction\src\screenshot_dataset.py", line 112, in __getitem__
img = self.transforms(img)
File "c:\2021-mcm-master\src\PyTorch-RCNN\ui-prediction\env\lib\site-packages\torchvision\transforms\transforms.py", line 60, in __call__
img = t(img)
File "c:\2021-mcm-master\src\PyTorch-RCNN\ui-prediction\env\lib\site-packages\torchvision\transforms\transforms.py", line 97, in __call__
return F.to_tensor(pic)
File "c:\2021-mcm-master\src\PyTorch-RCNN\ui-prediction\env\lib\site-packages\torchvision\transforms\functional.py", line 129, in to_tensor
np.array(pic, mode_to_nptype.get(pic.mode, np.uint8), copy=True)
TypeError: __array__() takes 1 positional argument but 2 were given
As #Phil suggested, downgrading Pillow from 8.3.0 to 8.2.0 solved the issue:
pip install pillow==8.2.0
Related
I have the following augmentations in my code
import albumentations as A
import torch
def get_train_transform():
transform = A.Compose([
A.RandomRotate90(),
A.Flip(),
A.Transpose(),
A.OneOf([
A.IAAAdditiveGaussianNoise(),
A.GaussNoise(),
], p=0.2),
A.OneOf([
A.MotionBlur(p=.2),
A.MedianBlur(blur_limit=3, p=0.1),
A.Blur(blur_limit=3, p=0.1),
], p=0.2),
A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.2),
A.OneOf([
A.OpticalDistortion(p=0.3),
A.GridDistortion(p=.1),
A.IAAPiecewiseAffine(p=0.3),
], p=0.2),
A.OneOf([
A.CLAHE(clip_limit=2),
A.IAASharpen(),
A.IAAEmboss(),
A.RandomBrightnessContrast(),
], p=0.3),
A.HueSaturationValue(p=0.3),
A.Normalize(mean=(0.5274, 0.4120, 0.3841), std=(0.1036, 0.0970, 0.0930)),
])
return transform
def get_val_transform():
transform = A.Compose([A.Normalize(mean=(0.5274, 0.4120, 0.3841), std=(0.1036, 0.0970, 0.0930))])
return transform
Here is how I call these transforms
train_dataset = MyDataset(train_data_path, 224, 224, train_labels_path, get_train_transform())
# The sampler, which takes into account an unbalanced dataset, is used only at the training step
train_loader= DataLoader(train_dataset, batch_size=8, num_workers=0, sampler=sampler)
# With validation data, the only transformation is the conversion into tensor
valid_dataset = MyDataset(test_data_path, 224, 224, test_labels_path, get_val_transform())
valid_loader = DataLoader(valid_dataset,batch_size=8, shuffle=False,num_workers=0)
However, once I run the code, it returns the following error:
File "dataset.py", line 41, in __getitem__
image_resized = self.transforms(image=image_resized)["image"]
File "/opt/conda/lib/python3.7/site-packages/albumentations/core/composition.py", line 205, in __call__
data = t(**data)
File "/opt/conda/lib/python3.7/site-packages/albumentations/core/composition.py",
line 309, in __call__
data = t(force_apply=True, **data)
File "/opt/conda/lib/python3.7/site-
packages/albumentations/core/transforms_interface.py", line 118, in __call__
return self.apply_with_params(params, **kwargs)
File "/opt/conda/lib/python3.7/site-
packages/albumentations/core/transforms_interface.py", line 131, in apply_with_params
res[key] = target_function(arg, **dict(params, **target_dependencies))
File "/opt/conda/lib/python3.7/site-packages/albumentations/augmentations/transforms.py", line 1310, in apply
return F.clahe(img, clip_limit, self.tile_grid_size)
File "/opt/conda/lib/python3.7/site-packages/albumentations/augmentations/utils.py", line 122, in wrapped_function
result = func(img, *args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/albumentations/augmentations/functional.py", line 452, in clahe
raise TypeError("clahe supports only uint8 inputs")
TypeError: clahe supports only uint8 inputs
I read from here that it seems that it is just a matter of putting the normalization as the last step of the augmentation, which is exactly what I am doing. So, what is supposed to be my mistake here?
I am working on transfer learning for an image classification task.
The training generator is as follows:
train_generator = train_datagen.flow_from_directory(
'/home/idu/Desktop/COV19D/train/',
color_mode = "grayscale",
target_size=(512, 512), # All images are 512 * 512
batch_size=batch_size,
classes = ['covid','non-covid'],
class_mode='binary')
The transferred model code is as follows:
SIZE = 512
VGG_model = VGG16(include_top=False, weights=None, input_shape=(SIZE, SIZE, 1))
for layer in VGG_model.layers:
layer.trainable = False
feature_extractor=VGG_model.predict(train_generator)
The last command throws the error:
Traceback (most recent call last):
File "<ipython-input-28-b9bad68819ec>", line 1, in <module>
feature_extractor=VGG_model.predict(train_generator)
File "/home/idu/.local/lib/python3.6/site-packages/keras/engine/training.py", line 1681, in predict
steps_per_execution=self._steps_per_execution)
File "/home/idu/.local/lib/python3.6/site-packages/keras/engine/data_adapter.py", line 1348, in get_data_handler
return DataHandler(*args, **kwargs)
File "/home/idu/.local/lib/python3.6/site-packages/keras/engine/data_adapter.py", line 1150, in __init__
model=model)
File "/home/idu/.local/lib/python3.6/site-packages/keras/engine/data_adapter.py", line 793, in __init__
peek, x = self._peek_and_restore(x)
File "/home/idu/.local/lib/python3.6/site-packages/keras/engine/data_adapter.py", line 850, in _peek_and_restore
peek = next(x)
File "/home/idu/.local/lib/python3.6/site-packages/keras_preprocessing/image/iterator.py", line 104, in __next__
return self.next(*args, **kwargs)
File "/home/idu/.local/lib/python3.6/site-packages/keras_preprocessing/image/iterator.py", line 116, in next
return self._get_batches_of_transformed_samples(index_array)
File "/home/idu/.local/lib/python3.6/site-packages/keras_preprocessing/image/iterator.py", line 231, in _get_batches_of_transformed_samples
x = img_to_array(img, data_format=self.data_format)
File "/home/idu/.local/lib/python3.6/site-packages/keras_preprocessing/image/utils.py", line 309, in img_to_array
x = np.asarray(img, dtype=dtype)
File "/home/idu/.local/lib/python3.6/site-packages/numpy/core/_asarray.py", line 83, in asarray
return array(a, dtype, copy=False, order=order)
TypeError: __array__() takes 1 positional argument but 2 were given
How can I overcome this error to do the feature exctraction?
Thank you.
I tried to downgrade tensorflow to 2.4, but that did not work. I downgraded my python version from 3.10.2 to 3.9.9 and re-installed scipy using the following command: python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose. This command solved the issue.
I am now running a Python program using Pytorch. I use my own dataset, not torch.data.dataset. I download data from a pickle file extracted from feature extraction. But the following errors appear:
Traceback (most recent call last):
File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demo-emotion.py", line 326, in <module>
fire.Fire(demo)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\fire\core.py", line 138, in Fire
component_trace = _Fire(component, args, parsed_flag_args, context, name)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\fire\core.py", line 468, in _Fire
target=component.__name__)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\fire\core.py", line 672, in _CallAndUpdateTrace
component = fn(*varargs, **kwargs)
File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demo-emotion.py", line 304, in demo
train(model,train_set1, valid_set=valid_set, test_set=test1, save=save, n_epochs=n_epochs,batch_size=batch_size,seed=seed)
File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demo-emotion.py", line 172, in train
n_epochs=n_epochs,
File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demo-emotion.py", line 37, in train_epoch
loader=np.asarray(list(loader))
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\dataloader.py", line 345, in __next__
data = self._next_data()
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\dataloader.py", line 385, in _next_data
data = self._dataset_fetcher.fetch(index) # may raise StopIteration
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\_utils\fetch.py", line 44, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\_utils\fetch.py", line 44, in <listcomp>
data = [self.dataset[idx] for idx in possibly_batched_index]
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\dataset.py", line 257, in __getitem__
return self.dataset[self.indices[idx]]
TypeError: 'DataLoader' object is not subscriptable
The code is:
train_set1 = Owndata()
train1, test1 = train_set1 .get_splits()
# prepare data loaders
train_dl = torch.utils.data.DataLoader(train1, batch_size=32, shuffle=True)
test_dl =torch.utils.data.DataLoader(test1, batch_size=1024, shuffle=False)
test_set1 = Owndata()
'''print('test_set# ',test_set)'''
if valid_size:
valid_set = Owndata()
indices = torch.randperm(len(train_set1))
train_indices = indices[:len(indices) - valid_size]
valid_indices = indices[len(indices) - valid_size:]
train_set1 = torch.utils.data.Subset(train_dl, train_indices)
valid_set = torch.utils.data.Subset(valid_set, valid_indices)
else:
valid_set = None
model = DenseNet(
growth_rate=growth_rate,
block_config=block_config,
num_classes=10,
small_inputs=True,
efficient=efficient,
)
train(model,train_set1, valid_set=valid_set, test_set=test1, save=save, n_epochs=n_epochs, batch_size=batch_size, seed=seed)
Any help is appreciated! Thanks a lot in advance!!
It is not the line giving you an error as it's the very last train function you are not showing.
You are confusing two things:
torch.utils.data.Dataset object is indexable (dataset[5] works fine for example). It is a simple object which defines how to get a single (usually single) sample of data.
torch.utils.data.DataLoader - non-indexable, only iterable, usually returns batches of data from above Dataset. Can work in parallel using num_workers. It's what you are trying to index while you should use dataset for that.
Please see PyTorch documentation about data to get a better grasp on how those work.
i work on thhe example based in this web and here is i got after this
jobs_train, jobs_test = jobs_df.randomSplit([0.6, 0.4])
>>> zuckerberg_train, zuckerberg_test = zuckerberg_df.randomSplit([0.6, 0.4])
>>> train_df = jobs_train.unionAll(zuckerberg_train)
>>> test_df = jobs_test.unionAll(zuckerberg_test)
>>> from pyspark.ml.classification import LogisticRegression
>>> from pyspark.ml import Pipeline
>>> from sparkdl import DeepImageFeaturizer
>>> featurizer = DeepImageFeaturizer(inputCol="image", outputCol="features", modelName="InceptionV3")
>>> lr = LogisticRegression(maxIter=20, regParam=0.05, elasticNetParam=0.3, labelCol="label")
>>> p = Pipeline(stages=[featurizer, lr])
>>> p_model = p.fit(train_df)
and this is appeared
2018-06-08 20:57:18.985543: I tensorflow/core/platform/cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
INFO:tensorflow:Froze 376 variables.
Converted 376 variables to const ops.
Using TensorFlow backend.
Using TensorFlow backend.
INFO:tensorflow:Froze 0 variables.
Converted 0 variables to const ops.
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/opt/spark/python/pyspark/ml/base.py", line 64, in fit
return self._fit(dataset)
File "/opt/spark/python/pyspark/ml/pipeline.py", line 106, in _fit
dataset = stage.transform(dataset)
File "/opt/spark/python/pyspark/ml/base.py", line 105, in transform
return self._transform(dataset)
File "/tmp/spark-74707b69-e8c9-498b-b0f2-b38828e5ad21/userFiles-ca1eb7cf-9785-441d-a098-54b62380bcee/databricks_spark-deep-learning-0.1.0-spark2.1-s_2.11.jar/sparkdl/transformers/named_image.py", line 159, in _transform
File "/opt/spark/python/pyspark/ml/base.py", line 105, in transform
return self._transform(dataset)
File "/tmp/spark-74707b69-e8c9-498b-b0f2-b38828e5ad21/userFiles-ca1eb7cf-9785-441d-a098-54b62380bcee/databricks_spark-deep-learning-0.1.0-spark2.1-s_2.11.jar/sparkdl/transformers/named_image.py", line 222, in _transform
File "/opt/spark/python/pyspark/ml/base.py", line 105, in transform
return self._transform(dataset)
File "/tmp/spark-74707b69-e8c9-498b-b0f2-b38828e5ad21/userFiles-ca1eb7cf-9785-441d-a098-54b62380bcee/databricks_spark-deep-learning-0.1.0-spark2.1-s_2.11.jar/sparkdl/transformers/tf_image.py", line 142, in _transform
File "/tmp/spark-74707b69-e8c9-498b-b0f2-b38828e5ad21/userFiles-ca1eb7cf-9785-441d-a098-54b62380bcee/databricks_tensorframes-0.2.8-s_2.11.jar/tensorframes/core.py", line 211, in map_rows
File "/tmp/spark-74707b69-e8c9-498b-b0f2-b38828e5ad21/userFiles-ca1eb7cf-9785-441d-a098-54b62380bcee/databricks_tensorframes-0.2.8-s_2.11.jar/tensorframes/core.py", line 132, in _map
File "/tmp/spark-74707b69-e8c9-498b-b0f2-b38828e5ad21/userFiles-ca1eb7cf-9785-441d-a098-54b62380bcee/databricks_tensorframes-0.2.8-s_2.11.jar/tensorframes/core.py", line 66, in _add_shapes
File "/tmp/spark-74707b69-e8c9-498b-b0f2-b38828e5ad21/userFiles-ca1eb7cf-9785-441d-a098-54b62380bcee/databricks_tensorframes-0.2.8-s_2.11.jar/tensorframes/core.py", line 35, in _get_shape
File "/home/sulistyo/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/tensor_shape.py", line 900, in as_list
raise ValueError("as_list() is not defined on an unknown TensorShape.")
ValueError: as_list() is not defined on an unknown TensorShape.
please kindly help, thanks
Use the following to read images and create your training & testing sets
from pyspark.sql.functions import lit
from sparkdl.image import imageIO
img_dir = "/PATH/TO/personalities/"
jobs_df = imageIO.readImagesWithCustomFn(img_dir + "/jobs",decode_f=imageIO.PIL_decode).withColumn("label", lit(1))
zuckerberg_df = imageIO.readImagesWithCustomFn(img_dir + "/zuckerberg", decode_f=imageIO.PIL_decode).withColumn("label", lit(0))
I am using the Keras ImageDataGenerator to process the inputs to my CNN. I want to basic preprocessing that scales the image pixels to values from -1 to 1 as it was done in the paper of the Mobilenet architecture.
My datagenerator only defines the preprocessing function:
train_datagen = ImageDataGenerator(
preprocessing_function=preprocess_input
)
My preprocess_input function:
def preprocess_input(img):
pix = np.asarray(img)
pix = pix.astype(np.float32)
pix = pix / 255.0
pix = pix * 2
return pix
This is giving me the follwing error:
Traceback (most recent call last): File "finetune_mobilenet.py",
line 206, in <module>
train(folder_train, folder_dev, './models/') File "finetune_mobilenet.py", line 150, in train
callbacks=callbacks_list) File "/usr/local/lib/python2.7/dist-packages/keras/legacy/interfaces.py",
line 91, in wrapper
return func(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/keras/engine/training.py",
line 2192, in fit_generator
generator_output = next(output_generator) File "/usr/local/lib/python2.7/dist-packages/keras/utils/data_utils.py",
line 584, in get
six.raise_from(StopIteration(e), e) File "/usr/local/lib/python2.7/dist-packages/six.py", line 737, in
raise_from
raise value StopIteration: 'tuple' object cannot be interpreted as an index
I also tried the original preprocessing function that is available for the Mobilenet architecture in Keras but that one also fails. Can you tell what I need to change to zero-center my image data?