I have following function. So I would like to take 2 fields and concatenate them but when my pipeline finishes it doesn´t work
the pipeline finishes as correct but when I see in bigquery the fields have not been concatenated
It will be great if u can help me.
it´s the code used in the function:
import apache_beam as beam
from ..tools import ProcessLogger
_logger = ProcessLogger()
class ConcatFieldsFn(beam.DoFn):
"""Concatena los valores de los campos especificados en una pCollection por un valor especificado"""
def __init__(self, process: str, data_name: str, parameters: dict):
# Configuracion del logger
self.logger_data_name = data_name
self.logger_process = process
self.logger_subprocess = "Concatenar valores"
_logger.data_name = self.logger_data_name
_logger.process = self.logger_process
_logger.subprocess = self.logger_subprocess
# Parametros del proceso
self._fields = [field.get("name") for field in parameters.get("fields", None)]
_logger.info(
f"Se aplica regla: {_logger.subprocess} con los parametros: {parameters}"
)
def process(self, element):
# Configuracion del logger
_logger.data_name = self.logger_data_name
_logger.process = self.logger_process
_logger.subprocess = self.logger_subprocess
for field in self._fields:
if element[field] != None:
try:
element[field] = "|".join(element[field])
except Exception as ex:
_logger.error(
f"No se pueden eliminar las letras de los campos seleccionados: {ex}"
)
return [element]
Related
I'm making a video game in upbge (which for this aspect is basically python) but to send the info between sockets it needs to be in bytes but i only found a way to do it turning it into a string, (because the list get's created with the game (i have a script that once a new entity appears it get's added to the list in the form of (Player-entity|data) but i want the data to variate between entities and i ran out of symbols to use to split it into lists so i'm trying to send the data as a list to be able to simply check it's components, but as it's a list inside of a list (and sometimes more) the bytearray won't work (at least i can't figure out how to make it work)
I'm unsure of how to summarize the code, but this is the code that I have now to generate that info
import socket
mi_socket = socket.socket()
mi_socket.bind(('127.0.0.1',55555))
mi_socket.listen()
TP = 0 #Total players
AP = 0 #Actual player
PL = ["host"] #Player List
AE = 0 #Actual entity
TE = 0 #Total entities
EL = ["|PlayerEntity$[Data]|"] #Entity list
PI = [] #Player Intel (Player name, Total Players, Player Code) (apartado uno en las comunicaciones)
order = None #Variable reservada para guardar comandos u identificaciones (ej: nombre de la entidad) en las comunicaciones
content = None #Variable reservada para almacenar la información recibida
incoming = 0 #Variable reservada para analizar el mensaje recibido de entrada
def login():
global AP
global TP
global PL
PJ = str(incoming).split("$")[1] #incoming Player
if AP <= TP:
if PJ!=str(PL[int(AP)]): #Nombre jugador que se conecta
AP+=1
login()
pass
else:
identity()
pass
if AP > TP:
PL.append(str("Player" + str(AP))) #Agregar jugador a la lista
TP = int(AP)
def identity(): #identifica de qué se trata la orden para responder en consecuencia, si se trata de entidad anota tmb la entidad en la lista
global TE
global AE
global EL
global AP
global PL
PJ = str(incoming).split("$")[1] # incoming Player
PE = str(incoming).split("$")[2] # incoming entity
if AE <= TE:
# Si nombre de jugador|nombre de objeto no és EL[AE]([1]|[2])
if str(str(PL[AP])+"-"+str(PE)) != str(str(EL[AE]).split("|")[1]).split("$")[0]:
AE+=1
identity()
pass
else:
EL[AE] = "|"+PL[AP]+"-"+PE+"$"+str(incoming).split("$")[3]+"|"
if AE > TE:
EL.append("|"+PL[AP]+"-"+PE+"$"+str(incoming).split("$")[3]+"|")
TE = int(AE)
def main():
global AP
global AE
global TE
global incoming
conexion, ip = mi_socket.accept()
incoming = conexion.recv(1024)
login()
conexion.send(str("#"+str(PL[AP])+"#"+str(EL[1:TE+1])+"#").encode())
AP=0
AE=0
conexion.close()
while True:
main()
We're making an exercise with pickle, and this code doesn't work as it is supposed to. Please help me, this is the code (some words are in Spanish because I'm from America):
import pickle
class persona:
def __init__(self, nombre, genero, edad):
self.nombre = nombre
self.genero = genero
self.edad = edad
print("se ha creado una persona nueva con el nombre de: ", self.nombre)
def __str__(self):
return "{} {} {}".format(self.nombre, self.genero, self.edad)
class listaPersonas:
personas = []
def __init__(self):
listaDePersonas = open("ficheroExterno", "ab+")
listaDePersonas.seek(0)
try:
self.personas = pickle.load(listaDePersonas)
print("Se cargaron {} personas del fichero externo".format(len(self.personas)))
except:
print("El fichero está vacío")
finally:
listaDePersonas.close()
del(listaDePersonas)
def agregarPersonas(self, p):
self.personas.append(p)
self.guardarPersonasEnFicheroExterno()
def mostrarPersonas(self):
for p in self.personas:
print(p)
def guardarPersonasEnFicheroExterno(self):
listaDePersonas = open("ficheroExterno", "wb")
pickle.dump(self.personas, listaDePersonas)
listaDePersonas.close()
del(listaDePersonas)
def mostrarInfoFicheroExterno(self):
print("La información sle fichero externo es la siguiente: ")
for p in self.personas:
print(p)
miLista = listaPersonas()
persona = persona("Sandra", "Femenino", 29)
miLista.agregarPersonas(persona)
miLista.mostrarInfoFicheroExterno()
and it throws that:
El fichero está vacío
se ha creado una persona nueva con el nombre de: Sandra
Traceback (most recent call last):
File "Guardado_permanente.py", line 54, in <module>
miLista.agregarPersonas(persona)
File "Guardado_permanente.py", line 34, in agregarPersonas
self.guardarPersonasEnFicheroExterno()
File "Guardado_permanente.py", line 42, in guardarPersonasEnFicheroExterno
pickle.dump(self.personas, listaDePersonas)
_pickle.PicklingError: Can't pickle <class '__main__.persona'>: it's not the same object as __main__.persona
***Repl Closed***
I have like 1 1/2 hour that I'm seeing this code and I'm trying to guess which is the problem, but the code is the same as my teacher's code. please, help me. I'm using Sublime text to code.
In this line, you have replaced your class persona with an instance of the class:
persona = persona("Sandra", "Femenino", 29)
pickle is trying to find the class definition for persona but can't, because it doesn't have a name anymore.
Don't try to use the same name for two things; only the last assignment counts. Standard style is to use CamelCase names for clasess, so you could name your class Persona instead.
I am implement a simple simulator of soccer with python using threads and lock, the app works fine but I have doubts in the way that implement the thread it seems to me that the first team has an advantage because is executing first.
def jugar(Equipo1, Equipo2):
# Busco las probabilidades de encajar por cada equipo
prob_encajar_eq1 = Equipo1.probabilidad_encajar()
prob_encajar_eq2 = Equipo2.probabilidad_encajar()
def jugar_equipo1(defensa_rival):
semaforo.acquire()
if Equipo1.hacer_pases():
Equipo1.shoot(defensa_rival)
semaforo.release()
def jugar_equipo2(defensa_rival):
semaforo.acquire()
if Equipo2.hacer_pases():
Equipo2.shoot(defensa_rival)
semaforo.release()
hilo_equipo1 = threading.Thread(name = 'hilo_eq1', target = jugar_equipo1, args = (prob_encajar_eq2,))
hilo_equipo2 = threading.Thread(name = 'hilo_eq2', target = jugar_equipo2, args = (prob_encajar_eq1,))
hilo_equipo1.start()
hilo_equipo2.start()
hilo_equipo1.join()
hilo_equipo2.join()
to make several attempts both teams, I do a cicle for a few seconds and inside the function jugar() which is the one that does the work with threads but here is were I have the doubts, because every time that jugar is executing the threads are declared again.
if __name__ == '__main__':
cargar_informacion()
eqA = Equipo(equipoA, ranking_eqA)
eqB = Equipo(equipoB, ranking_eqB)
probabilidades = porcenajes_ranking(ranking_eqA)
eqA.cargar_probabilidades(probabilidades)
probabilidades = porcenajes_ranking(ranking_eqB)
eqB.cargar_probabilidades(probabilidades)
starttime=time.time()
tiempo = 0
# creo la barra de progreso
bar = progressbar.ProgressBar(widgets=[
progressbar.Percentage(),
progressbar.Bar(),
], max_value=100).start()
# hacemos que el juego tarde aproximadamente 10seg en simularse.
while tiempo < 10:
time.sleep(0.3 - ((time.time() - starttime) % 0.3))
jugar(eqA,eqB)
tiempo = time.time() - starttime
bar += 2.8
bar.finish() # Para que finalice la barra de progreso
resultados_finales(eqA, eqB) # Mostramos el resultado final del partido.
I'm developing my API and currently I store one of my fields in spanish. Database is Postgres 9.5 (field type is varchar):
File Encoding : utf-8
Text field:
"text" varchar(65536) COLLATE "default"
When i return the value from text, I use flask_sqlalchemy to get my data.
class Serializer(object):
"""
Serialize information from database to a JSON Dictionary
"""
def serialize(self):
return {c: getattr(self, c) for c in inspect(self).attrs.keys()}
#staticmethod
def serialize_list(l):
return [m.serialize() for m in l]
class AutoSerialize(object):
"""
Mixin for retrieving public fields of model in json-compatible format'
"""
__public__ = None
def get_public(self, exclude=(), extra=()):
"Returns model's PUBLIC data for jsonify"
data = {}
keys = self._sa_instance_state.attrs.items()
public = self.__public__ + extra if self.__public__ else extra
for k, field in keys:
if public and k not in public: continue
if k in exclude: continue
value = self._serialize(field.value)
if value:
data[k] = value
return data
#classmethod
def _serialize(cls, value, follow_fk=False):
if type(value) in (datetime,):
ret = value.isoformat()
elif hasattr(value, '__iter__'):
ret = []
for v in value:
ret.append(cls._serialize(v))
elif AutoSerialize in value.__class__.__bases__:
ret = value.get_public()
else:
ret = value
return ret
My field in my Model is defined as follows and my class inherits Serializer and AutoSerialize:
description = Column(String(65536), nullable=True)
This is how I return my values to API client:
articles = Model.Bits.query.order_by(Model.Bits.publishedAt.desc()).limit(10).all()
if articles:
log.info('api() | %d Articles found ' % len(articles))
response = []
values = ['author', 'title', 'description', 'url', 'urlToImage', 'publishedAt']
response = [{value: getattr(d, value) for value in values} for d in articles]
return jsonify(articles=response, status='ok', source='my_source', sortBy='latest')
My response looks like this using curl:
{
"author": "Bros Lopez",
"description": "Spotify quiere ser m\u00e1s competitivo en su servicio de recomendaciones de contenido frente a marcas como Apple Music y Pandora. La empresa anunci\u00f3 la compra de la startup francesa Niland, la cual utiliza un sistema de inteligencia artificial para b\u00fasqueda y recomendaciones de contenido. Con 50 millones de usuarios activos Spotify quiere ser m\u00e1s rentable, pues a pesar de que el a\u00f1o pasado gener\u00f3 $3.1 mmdd en ventas, su margen bruto fue del 14%, pagando cerca de 2.7 mmdd en sellos discogr\u00e1ficos y editoriales. Por su parte, Pandora, unos de los principales competidores de Spotify, podr\u00eda ser adquirida por la empresa de radiodifusi\u00f3n SiriusXM, quien el a\u00f1o pasado le hizo una propuesta de compra por $3.4 mmdd. More Info",
"publishedAt": "Fri, 19 May 2017 20:00:00 GMT",
"title": "\u00bfPandora o Spotify?",
"url": "http://www.cnbc.com/2017/05/18/spotify-buys-niland-french-ai-music-startup.html",
"urlToImage": "https://ci4.googleusercontent.com/proxy/TWmEZRwlpPQrjs4HGZGx2041GryyquO7CjSR0oVBK-JUy4Xv3qHSiDow056iW8DV059chC93zFeXc4GVHKnzPpweUy-JzamK-l9pkW-Hgl1PnOun5s4XsE7K2NXBJljp-1Ltf5jyjfcn4j63Hv68FdFuqsw5UNTFBKkFug0=s0-d-e1-ft#https://gallery.mailchimp.com/f82949535ab2aab4bafde98f6/images/1f0dc47c-358b-4625-8744-105ffccfed98.jpg"
}
Is the encoding correct? I tried different client and characters are displayed correctly, so not sure if it is up to the client to display info properly or server.
It is a client's job to parse such characters, which curl is obviously not doing "out-of-the-box". Depending on OS/shell/encoding you are using, there might be some ways (or the others) to pipe the response to some other command which would parse those characters or some similar approach.
I'm testing a query with python and sqlite3. First method works fine, but
second is not working. It is about the defined type of variable containing the resgisters in DB:
import sqlite3
def insertar():
db1=sqlite3.connect('tabla.db')
print("Estas en la funcion insertar")
nombre1=raw_input("Escribe el titulo de la novela: ")
autor1=raw_input("Escribe el autor de la novela: ")
year1=str(input("Digita el any de la novela: "))
consulta=db1.cursor()
strConsulta = "insert into tabla(nombre, autor, year) values\
('"+nombre1+"','"+autor1+"','"+year1+"')"
print(strConsulta)
consulta.execute(strConsulta)
consulta.close()
db1.commit()
db1.close()
def consultar():
db2 = sqlite3.connect("tabla.db")
print("Estas en la funcion insertar")
db2row_factory = sqlite3.Row
consulta = db2.cursor()
consulta.execute("select * from tabla")
filas = consulta.fetchall()
lista = []
for fila in filas:
s = {}
s['nombre'] = fila['nombre']
s['autor'] = fila['autor']
s['year'] = str(fila['year'])
lista.append(s)
consulta.close()
db2.close()
return(lista)
#consultar()
def menu():
Opcion= input("\nIngresa la opcion deseada\n1.Inserta un valor en la tabla\n2.Consultar los valores de la tabla\n")
if Opcion==1:
insertar()
menu()
elif Opcion==2:
ListaNovelas = consultar()
for novela in ListaNovelas:
print(novela['nombre'],novela['autor'],novela['year'])
menu()
menu()
I get this error while testing the second method consultar().
$ python file.py
Ingresa la opcion deseada
1.Inserta un valor en la tabla
2.Consultar los valores de la tabla
2
Estas en la funcion insertar
Traceback (most recent call last):
File "insertar.py", line 56, in <module>
menu()
File "insertar.py", line 51, in menu
ListaNovelas = consultar()
File "insertar.py", line 33, in consultar
s['nombre'] = fila['nombre']
TypeError: tuple indices must be integers, not str
db2row_factory = sqlite3.Row
This is the problematic line. Instead you meant to set the row_factory factory on the db2 connection instance:
db2.row_factory = sqlite3.Row
Then, all the fetched rows would be now sqlite3.Row instances having dictionary-like access to field values.