How do i select only certain rows based on label in pandas? - python
Here is my function:
def get_historical_closes(ticker, start_date, end_date):
my_dir = '/home/manish/Desktop/Equity/subset'
os.chdir(my_dir)
dfs = []
for files in glob.glob('*.txt'):
dfs.append(pd.read_csv(files, names = ['Ticker', 'Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Null'], parse_dates = [1]))
p = pd.concat(dfs)
d = p.reset_index(['Date', 'Ticker', 'Close'])
pivoted = d.pivot_table(index = ['Date'], columns =['Ticker'])
pivoted.columns = pivoted.columns.droplevel(0)
return pivoted
closes = get_historical_closes(['LT' or 'HDFC'or 'ACC'], '1999-01-01', '2014-12-31')
My problem is I just want to get data for a few rows namely, data for LT, HDFC and ACC for all the dates, but when I execute the function, I am getting data for all the rows (approx. 1500 nos.)
How can I slice the dataframe, so that I get only selected rows and not the entire dataframe?
Raw input data is a collection of text files as so:
20MICRONS,20150401,36.5,38.95,35.8,37.35,64023,0
3IINFOTECH,20150401,5.9,6.3,5.8,6.2,1602365,0
3MINDIA,20150401,7905,7905,7850,7879.6,310,0
8KMILES,20150401,710.05,721,706,712.9,20196,0
A2ZINFRA,20150401,15.5,16.55,15.2,16,218219,0
AARTIDRUGS,20150401,648.95,665.5,639.65,648.25,42927,0
AARTIIND,20150401,348,349.4,340.3,341.85,122071,0
AARVEEDEN,20150401,42,42.9,41.55,42.3,627,0
ABAN,20150401,422,434.3,419,429.1,625857,0
ABB,20150401,1266.05,1284,1266,1277.45,70294,0
ABBOTINDIA,20150401,3979.25,4009.95,3955.3,3981.25,2677,0
ABCIL,20150401,217.8,222.95,217,221.65,11583,0
ABGSHIP,20150401,225,225,215.3,220.2,237737,0
ABIRLANUVO,20150401,1677,1677,1639.25,1666.7,106336,0
ACC,20150401,1563.7,1591.3,1553.2,1585.9,176063,0
ACCELYA,20150401,932,953.8,923,950.5,4297,0
ACE,20150401,40.1,41.7,40.05,41.15,356130,0
ACROPETAL,20150401,2.75,3,2.7,2.85,33380,0
ADANIENT,20150401,608.8,615.8,603,612.4,868006,0
ADANIPORTS,20150401,308.45,312.05,306.1,310.95,1026200,0
ADANIPOWER,20150401,46.7,48,46.7,47.75,3015649,0
ADFFOODS,20150401,60.5,60.5,58.65,59.75,23532,0
ADHUNIK,20150401,20.95,21.75,20.8,21.2,149431,0
ADORWELD,20150401,224.9,224.9,215.65,219.2,2743,0
ADSL,20150401,19,20,18.7,19.65,35053,0
ADVANIHOTR,20150401,43.1,43.1,43,43,100,0
ADVANTA,20150401,419.9,430.05,418,428,16206,0
AEGISCHEM,20150401,609,668,600,658.4,264828,0
AFL,20150401,65.25,70,65.25,68.65,9507,0
AGARIND,20150401,95,100,87.25,97.45,14387,0
AGCNET,20150401,91.95,93.75,91.4,93,2453,0
AGRITECH,20150401,5.5,6.1,5.5,5.75,540,0
AGRODUTCH,20150401,2.7,2.7,2.6,2.7,451,0
AHLEAST,20150401,196,202.4,185,192.25,357,0
AHLUCONT,20150401,249.5,258.3,246,251.3,44541,0
AHLWEST,20150401,123.9,129.85,123.9,128.35,688,0
AHMEDFORGE,20150401,229.5,237.35,228,231.45,332680,0
AIAENG,20150401,1268,1268,1204.95,1214.1,48950,0
AIL,20150401,735,747.9,725.1,734.8,31780,0
AJANTPHARM,20150401,1235,1252,1207.05,1223.3,126442,0
AJMERA,20150401,118.7,121.9,117.2,118.45,23005,0
AKSHOPTFBR,20150401,14.3,14.8,14.15,14.7,214028,0
AKZOINDIA,20150401,1403.95,1412,1392,1400.7,17115,0
ALBK,20150401,99.1,101.65,99.1,101.4,2129046,0
ALCHEM,20150401,27.9,32.5,27.15,31.6,32338,0
ALEMBICLTD,20150401,34.6,36.7,34.3,36.45,692688,0
ALICON,20150401,280,288,279.05,281.05,5937,0
ALKALI,20150401,31.6,34.2,31.6,33.95,4663,0
ALKYLAMINE,20150401,314,334,313.1,328.8,1515,0
ALLCARGO,20150401,317,323.5,315,319.15,31056,0
ALLSEC,20150401,21.65,22.5,21.6,21.6,435,0
ALMONDZ,20150401,10.6,10.95,10.5,10.75,23600,0
ALOKTEXT,20150401,7.5,8.2,7.4,7.95,8145264,0
ALPA,20150401,11.85,11.85,10.75,11.8,3600,0
ALPHAGEO,20150401,384.3,425.05,383.95,419.75,13308,0
ALPSINDUS,20150401,1.85,1.85,1.85,1.85,1050,0
ALSTOMT&D,20150401,585.85,595,576.65,588.4,49234,0
AMARAJABAT,20150401,836.5,847.75,831,843.9,121150,0
AMBIKCO,20150401,790,809,780.25,802.6,4879,0
AMBUJACEM,20150401,254.95,261.4,253.4,260.25,1346375,0
AMDIND,20150401,20.5,22.75,20.5,22.3,693,0
AMRUTANJAN,20150401,480,527.05,478.35,518.3,216407,0
AMTEKAUTO,20150401,144.5,148.45,144.2,147.45,552874,0
AMTEKINDIA,20150401,55.6,58.3,55.1,57.6,700465,0
AMTL,20150401,13.75,14.45,13.6,14.45,2111,0
ANANTRAJ,20150401,39.9,40.3,39.35,40.05,376564,0
ANDHRABANK,20150401,78.35,80.8,78.2,80.55,993038,0
ANDHRACEMT,20150401,8.85,9.3,8.75,9.1,15848,0
ANDHRSUGAR,20150401,92.05,98.95,91.55,96.15,11551,0
ANGIND,20150401,36.5,36.9,35.6,36.5,34758,0
ANIKINDS,20150401,22.95,24.05,22.95,24.05,1936,0
ANKITMETAL,20150401,2.85,3.25,2.85,3.15,29101,0
ANSALAPI,20150401,23.45,24,23.45,23.8,76723,0
ANSALHSG,20150401,29.9,29.9,28.75,29.65,7748,0
ANTGRAPHIC,20150401,0.1,0.15,0.1,0.15,23500,0
APARINDS,20150401,368.3,375.6,368.3,373.45,2719,0
APCOTEXIND,20150401,505,505,481.1,495.85,3906,0
APLAPOLLO,20150401,411.5,434,411.5,428.65,88113,0
APLLTD,20150401,458.9,464,450,454.7,72075,0
APOLLOHOSP,20150401,1351,1393.85,1351,1390,132827,0
APOLLOTYRE,20150401,169.65,175.9,169,175.2,3515274,0
APOLSINHOT,20150401,195,197,194.3,195.2,71,0
APTECHT,20150401,57.6,61,57,59.7,206475,0
ARCHIDPLY,20150401,32.95,35.8,32.5,35.35,103036,0
ARCHIES,20150401,19.05,19.4,18.8,19.25,46840,0
ARCOTECH,20150401,342.5,350,339.1,345.2,44142,0
ARIES,20150401,106.75,113.9,105,112.7,96825,0
ARIHANT,20150401,43.5,50,43.5,49.3,1647,0
AROGRANITE,20150401,61.5,62,59.55,60.15,2293,0
ARROWTEX,20150401,25.7,27.8,25.1,26.55,17431,0
ARSHIYA,20150401,39.55,41.5,39,40,69880,0
ARSSINFRA,20150401,34.65,36.5,34.6,36.3,71442,0
ARVIND,20150401,260.85,268.2,259,267.2,1169433,0
ARVINDREM,20150401,15.9,17.6,15.5,17.6,5407412,0
ASAHIINDIA,20150401,145,145,141,142.45,16240,0
ASAHISONG,20150401,113,116.7,112.15,115.85,5475,0
ASAL,20150401,45.8,45.8,38,43.95,7429,0
ASHAPURMIN,20150401,74,75.4,74,74.05,36406,0
ASHIANA,20150401,248,259,246.3,249.5,21284,0
ASHIMASYN,20150401,8.4,8.85,8.05,8.25,3253,0
ASHOKA,20150401,175.1,185.4,175.1,183.75,1319134,0
ASHOKLEY,20150401,72.7,74.75,72.7,74.05,17233199,0
ASIANHOTNR,20150401,104.45,107.8,101.1,105.15,780,0
ASIANPAINT,20150401,810,825.9,803.5,821.7,898480,0
ASIANTILES,20150401,116.25,124.4,116.25,123.05,31440,0
ASSAMCO,20150401,4.05,4.3,4.05,4.3,476091,0
ASTEC,20150401,148.5,154.5,146,149.2,322308,0
ASTRAL,20150401,447.3,451.3,435.15,448.6,64889,0
ASTRAMICRO,20150401,146.5,151.9,145.2,150.05,735681,0
ASTRAZEN,20150401,908,940.95,908,920.35,3291,0
ATFL,20150401,635,648,625.2,629.25,6202,0
ATLANTA,20150401,67.2,71,67.2,68.6,238683,0
ATLASCYCLE,20150401,203.9,210.4,203,208.05,25208,0
ATNINTER,20150401,0.2,0.2,0.2,0.2,1704,0
ATUL,20150401,1116,1160,1113,1153.05,32969,0
ATULAUTO,20150401,556.55,576.9,555.9,566.25,59117,0
AURIONPRO,20150401,192.3,224.95,191.8,217.55,115464,0
AUROPHARMA,20150401,1215,1252,1215,1247.4,1140111,0
AUSOMENT,20150401,22.6,22.6,21.7,21.7,2952,0
AUSTRAL,20150401,0.5,0.55,0.5,0.5,50407,0
AUTOAXLES,20150401,834.15,834.15,803,810.2,4054,0
AUTOIND,20150401,60,65,59.15,63.6,212036,0
AUTOLITIND,20150401,36,39,35.2,37.65,14334,0
AVTNPL,20150401,27,28,26.7,27.9,44803,0
AXISBANK,20150401,557.7,572,555.25,569.65,3753262,0
AXISCADES,20150401,335.4,345,331.4,339.65,524538,0
AXISGOLD,20150401,2473.95,2493,2461.1,2483.15,138,0
BAFNAPHARM,20150401,29.95,31.45,29.95,30.95,21136,0
BAGFILMS,20150401,3.05,3.1,2.9,3,31278,0
BAJAJ-AUTO,20150401,2027.05,2035,2002.95,2019.8,208545,0
BAJAJCORP,20150401,459,482,454,466.95,121972,0
BAJAJELEC,20150401,230,234.8,229,232.4,95432,0
BAJAJFINSV,20150401,1412,1447.5,1396,1427.55,44811,0
BAJAJHIND,20150401,14.5,14.8,14.2,14.6,671746,0
BAJAJHLDNG,20150401,1302.3,1329.85,1285.05,1299.9,24626,0
BAJFINANCE,20150401,4158,4158,4062.2,4140.05,12923,0
BALAJITELE,20150401,65.75,67.9,65.3,67.5,47063,0
BALAMINES,20150401,81.5,83.5,81.5,83.45,6674,0
BALKRISIND,20150401,649,661,640,655,16919,0
BALLARPUR,20150401,13.75,13.95,13.5,13.9,271962,0
BALMLAWRIE,20150401,568.05,580.9,562.2,576.75,17423,0
BALPHARMA,20150401,68.9,74.2,67.1,68.85,84178,0
BALRAMCHIN,20150401,50.95,50.95,49.3,50,84400,0
BANARBEADS,20150401,33,39.5,33,39.25,1077,0
BANARISUG,20150401,834.7,855,820,849.85,618,0
BANCOINDIA,20150401,105,107.5,103.25,106.8,11765,0
BANG,20150401,6.2,6.35,6.1,6.35,9639,0
BANKBARODA,20150401,162.75,170.4,162.05,168.9,2949846,0
BANKBEES,20150401,1813.45,1863,1807,1859.78,19071,0
BANKINDIA,20150401,194.6,209.8,194.05,205.75,3396490,0
BANSWRAS,20150401,65,65,60.1,63.9,6238,0
BARTRONICS,20150401,11.45,11.85,11.35,11.6,109658,0
BASF,20150401,1115,1142,1115,1124.65,14009,0
BASML,20150401,184,192,183.65,191.6,642,0
BATAINDIA,20150401,1095,1104.9,1085,1094.7,137166,0
BAYERCROP,20150401,3333,3408.3,3286.05,3304.55,8839,0
BBL,20150401,627.95,641.4,622.2,629.8,5261,0
BBTC,20150401,441,458,431.3,449.15,141334,0
BEDMUTHA,20150401,16.85,18,16.25,17.95,16412,0
BEL,20150401,3355,3595,3350,3494.2,582755,0
BEML,20150401,1100,1163.8,1086,1139.2,631231,0
BEPL,20150401,22.1,22.45,21.15,22.3,5459,0
BERGEPAINT,20150401,209.3,216.9,208.35,215.15,675963,0
BFINVEST,20150401,168.8,176.8,159.5,172.7,113352,0
BFUTILITIE,20150401,707.4,741,702.05,736.05,1048274,0
BGLOBAL,20150401,2.9,3.05,2.9,3.05,16500,0
BGRENERGY,20150401,117.35,124,117.35,122.3,207979,0
BHAGYNAGAR,20150401,17.9,17.9,16.95,17.5,1136,0
BHARATFORG,20150401,1265.05,1333.1,1265.05,1322.6,704419,0
BHARATGEAR,20150401,73.5,77.7,72.7,75.9,13730,0
BHARATRAS,20150401,810,840,800,821.4,981,0
BHARTIARTL,20150401,393.3,404.85,393.05,402.3,5494883,0
BHEL,20150401,235.8,236,229.6,230.7,3346075,0
BHUSANSTL,20150401,65.15,67.9,63.65,64,1108540,0
BIL,20150401,401.3,422,401.3,419.35,2335,0
BILENERGY,20150401,0.8,0.95,0.8,0.95,8520,0
BINANIIND,20150401,90.55,93.95,90.2,93.3,27564,0
BINDALAGRO,20150401,23.4,23.4,22.25,22.8,111558,0
BIOCON,20150401,472.5,478.85,462.7,466.05,1942983,0
BIRLACORPN,20150401,415,420,402.8,414.7,11345,0
BIRLACOT,20150401,0.05,0.1,0.05,0.1,439292,0
BIRLAERIC,20150401,52.3,54.45,52.15,53.7,9454,0
BIRLAMONEY,20150401,24.35,28.85,23.9,28.65,78710,0
BLBLIMITED,20150401,3.7,3.7,3.65,3.65,550,0
BLISSGVS,20150401,128,132.55,124.3,126.15,261958,0
BLKASHYAP,20150401,13.7,15.15,13.7,14.15,118455,0
BLUEDART,20150401,7297.35,7315,7200,7285.55,2036,0
BLUESTARCO,20150401,308.75,315,302,311.35,19046,0
BLUESTINFO,20150401,199,199.9,196.05,199.45,1268,0
BODALCHEM,20150401,34.5,34.8,33.05,34.65,65623,0
BOMDYEING,20150401,64,66.3,63.7,65.95,1168851,0
BOSCHLTD,20150401,25488,25708,25201,25570.7,16121,0
BPCL,20150401,810.95,818,796.5,804.2,1065969,0
BPL,20150401,30.55,32.5,30.55,31.75,116804,0
BRFL,20150401,146,147.9,142.45,144.3,7257,0
BRIGADE,20150401,143.8,145.15,140.25,144.05,36484,0
BRITANNIA,20150401,2155.5,2215.3,2141.35,2177.55,245908,0
BROADCAST,20150401,3.35,3.5,3.3,3.3,4298,0
BROOKS,20150401,38.4,39.5,38.4,39.3,19724,0
BSELINFRA,20150401,1.9,2.15,1.85,2.05,97575,0
BSL,20150401,29.55,31.9,27.75,31,9708,0
BSLGOLDETF,20150401,2535,2535,2501.5,2501.5,122,0
BSLIMITED,20150401,27.5,27.5,25.45,27.15,728818,0
BURNPUR,20150401,9.85,9.85,9.1,9.15,144864,0
BUTTERFLY,20150401,190.95,194,186.1,192.35,25447,0
BVCL,20150401,17.25,17.7,16.5,17.7,9993,0
CADILAHC,20150401,1755,1796.8,1737.05,1790.15,302149,0
CAIRN,20150401,213.85,215.6,211.5,213.35,841463,0
CAMLINFINE,20150401,89.5,91.4,87.5,91.1,32027,0
CANBK,20150401,366.5,383.8,365.15,381,1512605,0
CANDC,20150401,20.6,24.6,20.6,23.25,9100,0
CANFINHOME,20150401,611.1,649.95,611.1,644.7,72233,0
CANTABIL,20150401,47.6,50.5,47.6,50.25,5474,0
CAPF,20150401,398.85,427,398,421.75,224074,0
CAPLIPOINT,20150401,1020,1127.8,1020,1122.65,108731,0
CARBORUNIV,20150401,191.05,197,188.35,190,42681,0
CAREERP,20150401,151.9,156.6,149,153.25,26075,0
CARERATING,20150401,1487,1632.75,1464,1579.2,65340,0
CASTROLIND,20150401,476,476.25,465.1,467.3,185850,0
CCCL,20150401,4.2,4.7,4.2,4.65,47963,0
CCHHL,20150401,10.8,11,10.4,10.8,69325,0
CCL,20150401,178.35,185.9,176,184.3,244917,0
CEATLTD,20150401,805.25,830.8,785.75,826.7,501415,0
CEBBCO,20150401,18.3,20.25,18.1,19.85,40541,0
CELEBRITY,20150401,11.5,12.5,11.5,12.1,5169,0
CELESTIAL,20150401,59.9,61.8,59.5,60.05,128386,0
CENTENKA,20150401,152,159.9,148.2,157.1,16739,0
CENTEXT,20150401,1.5,1.5,1.2,1.25,19308,0
CENTRALBK,20150401,106,107.2,104.3,106.3,992782,0
CENTUM,20150401,756.85,805,756.8,801.9,26848,0
CENTURYPLY,20150401,234,245,234,243.45,367540,0
CENTURYTEX,20150401,633.6,682.4,631,675.35,3619413,0
CERA,20150401,2524.75,2524.75,2470,2495.3,6053,0
CEREBRAINT,20150401,15.6,16.2,14.65,14.8,348478,0
CESC,20150401,604.95,613.4,595.4,609.75,294334,0
CGCL,20150401,173,173,173,173,9,0
CHAMBLFERT,20150401,70.2,73.4,70.2,72.65,2475030,0
CHEMFALKAL,20150401,72.8,77,72,76.3,1334,0
CHENNPETRO,20150401,69,70.35,68.3,68.95,160576,0
CHESLINTEX,20150401,10.1,10.1,8.75,9.4,1668,0
CHOLAFIN,20150401,599.85,604,582.15,598.2,23125,0
CHROMATIC,20150401,3.4,4.05,3,3.3,63493,0
CIGNITITEC,20150401,433,444.95,432,440,32923,0
CIMMCO,20150401,92,94.05,91,94.05,19931,0
CINELINE,20150401,14.5,14.95,14.5,14.9,4654,0
CINEVISTA,20150401,3.3,3.3,3.3,3.3,10,0
CIPLA,20150401,714,716.5,703.85,709.6,1693796,0
CLASSIC,20150401,1.5,1.55,1.45,1.45,7770,0
CLNINDIA,20150401,824.7,837.9,819,828.8,6754,0
CLUTCHAUTO,20150401,13.75,13.75,13.6,13.6,1414,0
CMAHENDRA,20150401,9.35,9.5,8.9,9.15,1005172,0
CMC,20150401,1925.85,1925.85,1891,1907.25,153068,0
CNOVAPETRO,20150401,20,22.75,17.1,22.75,1656,0
COALINDIA,20150401,362.9,364.25,358,363,1428949,0
COLPAL,20150401,2003.4,2009.9,1990.05,2002.5,92909,0
COMPUSOFT,20150401,9.4,10.05,9,9.7,15083,0
CONCOR,20150401,1582.35,1627.3,1561,1582.85,182280,0
CONSOFINVT,20150401,36.55,40,36.5,40,439,0
CORDSCABLE,20150401,25.55,28,24.1,25.8,15651,0
COREEDUTEC,20150401,8,8.85,7.6,8.4,890455,0
COROMANDEL,20150401,268.5,271.35,266.15,268.35,42173,0
CORPBANK,20150401,52.5,55,52.05,54.1,1141752,0
COSMOFILMS,20150401,76.9,80,76.2,79.25,21020,0
COUNCODOS,20150401,1.2,1.2,1.2,1.2,2850,0
COX&KINGS,20150401,323,324.85,316.5,317.8,76998,0
CPSEETF,20150401,24.2,24.37,24.08,24.34,180315,0
CREATIVEYE,20150401,3.4,3.6,2.8,3.45,8545,0
CRISIL,20150401,2049,2052.45,2000,2030.7,3928,0
CROMPGREAV,20150401,164.85,167.4,163.2,166.1,2739478,0
CTE,20150401,18.55,18.55,16.85,17.05,8260,0
CUB,20150401,97.35,98.75,96.4,98.3,182702,0
CUMMINSIND,20150401,879,900.95,874.75,889.9,358652,0
CURATECH,20150401,10.8,11,9.75,10,755,0
CYBERTECH,20150401,28.5,33.45,28.1,33.4,103549,0
CYIENT,20150401,509.9,515,495.1,514.1,30415,0
DAAWAT,20150401,105,112.25,99.5,108.4,26689,0
DABUR,20150401,266.5,268.5,264.65,266.55,642177,0
DALMIABHA,20150401,428.15,439.9,422.5,432.65,9751,0
DALMIASUG,20150401,17.5,17.5,16.45,17.15,12660,0
DATAMATICS,20150401,66.5,75,66,72.15,119054,0
DBCORP,20150401,378,378,362.6,369.45,8799,0
DBREALTY,20150401,67,67.15,65.8,66.3,212297,0
DBSTOCKBRO,20150401,47.6,47.65,47.45,47.55,24170,0
DCBBANK,20150401,110.95,114.95,110.15,114.45,935858,0
DCM,20150401,84.5,88.75,84.1,87,34747,0
DCMSHRIRAM,20150401,107.95,114.3,107.95,112.8,29474,0
DCW,20150401,16.75,17.2,16.65,17.15,270502,0
DECCANCE,20150401,310.05,323.9,310.05,321.55,446,0
DECOLIGHT,20150401,1.45,1.45,1.4,1.4,1100,0
DEEPAKFERT,20150401,140,144,138.25,139.95,162156,0
DEEPAKNTR,20150401,68,70.65,66.4,69.95,8349,0
DEEPIND,20150401,46.6,54.4,46.3,51.9,52130,0
DELTACORP,20150401,79.95,82.75,79.75,82.35,889247,0
DELTAMAGNT,20150401,36.6,37.45,36.6,37.45,60,0
DEN,20150401,121.45,127,121.2,122.4,59512,0
DENABANK,20150401,50.8,51.5,50.1,51.35,376680,0
DENORA,20150401,136.7,136.7,131.05,133.6,743,0
DHAMPURSUG,20150401,36.8,36.95,34.85,36.35,38083,0
DHANBANK,20150401,30.8,32.1,30.5,31.75,195779,0
DHANUKA,20150401,690,690,652,660.15,24958,0
DHARSUGAR,20150401,14.15,14.7,13.8,14.45,1748,0
DHFL,20150401,468.9,474.9,461.6,467.85,448551,0
DHUNINV,20150401,97.15,103,94.5,99.85,15275,0
DIAPOWER,20150401,44.9,45.95,43.3,45.55,126085,0
DICIND,20150401,343,347,341,341.95,7745,0
DIGJAM,20150401,8,8.15,7.75,8.05,96467,0
DISHMAN,20150401,168,172.65,164.7,171.8,778414,0
DISHTV,20150401,82.2,84.85,81.35,84.15,5845850,0
DIVISLAB,20150401,1770.1,1809,1770.1,1802.35,68003,0
DLF,20150401,157,160.9,156.2,159.7,3098216,0
DLINKINDIA,20150401,165.05,168,162.2,164.75,22444,0
DOLPHINOFF,20150401,120.8,134.4,119.5,130.2,190716,0
DONEAR,20150401,15,15.95,14.5,15.35,679,0
DPL,20150401,46.6,49,44,45.45,25444,0
DPSCLTD,20150401,17.15,17.15,16.55,16.85,916,0
DQE,20150401,24.3,24.8,22.75,23.1,57807,0
DRDATSONS,20150401,5.8,6.1,5.7,6,2191357,0
DREDGECORP,20150401,374.9,403,372.65,393.4,106853,0
DRREDDY,20150401,3541,3566.8,3501.7,3533.65,282785,0
DSKULKARNI,20150401,77.6,77.6,74,77.1,3012,0
DSSL,20150401,9.5,9.5,9.5,9.5,50,0
DTIL,20150401,206.95,231.75,205.95,219.05,1437,0
DUNCANSLTD,20150401,15.55,16.3,15.3,15.85,740,0
DWARKESH,20150401,21,21,19.85,20.7,9410,0
DYNAMATECH,20150401,3868,4233,3857.1,3920.55,59412,0
DYNATECH,20150401,2.85,3,2.85,3,3002,0
EASTSILK,20150401,1.55,1.85,1.55,1.75,9437,0
EASUNREYRL,20150401,40.05,43,40.05,42.55,21925,0
ECEIND,20150401,136,148,127,133.85,43034,0
ECLERX,20150401,1603.8,1697,1595,1600.65,123468,0
EDELWEISS,20150401,63.65,67.5,63,66.6,451255,0
EDL,20150401,23.9,25,23.9,24.4,7799,0
EDUCOMP,20150401,12.45,13.55,12.35,13.55,499009,0
EICHERMOT,20150401,15929,16196.95,15830.05,16019.5,45879,0
EIDPARRY,20150401,174.05,175.8,168.65,171.2,56813,0
EIHAHOTELS,20150401,228,232.8,225,228,85,0
EIHOTEL,20150401,107.25,110,107.25,109.5,57306,0
EIMCOELECO,20150401,399,409.5,399,409.5,184,0
EKC,20150401,9.35,11.15,9.35,11.05,350782,0
ELAND,20150401,14.3,16.45,14.3,16.25,191406,0
ELDERPHARM,20150401,90.5,91.5,89.45,91.5,23450,0
ELECON,20150401,66.5,76.2,66.25,74.45,6045416,0
ELECTCAST,20150401,19.8,20.55,18.9,19.4,1956889,0
ELECTHERM,20150401,25.9,25.9,22.2,24,14611,0
ELGIEQUIP,20150401,147.5,150.4,146.4,150,9475,0
....
ZENITH, 20150401,...
I use EdChum code from his comment and add some clarification. I think the main problem is d is output dataframe d cannot be looped in cycle for, if you need one output from all *.txt files.
import pandas as pd
import glob
def get_historical_closes(ticker, start_date, end_date):
dfs = []
#create empty df for output
d = pd.DataFrame()
#glob can use path with *.txt - see http://stackoverflow.com/a/3215392/2901002
for files in glob.glob('/home/manish/Desktop/Equity/subset/*.txt'):
#added index_col for multiindex df
dfs.append(pd.read_csv(files, index_col=['Date', 'Ticker', 'Close'], names = ['Ticker', 'Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Null'], parse_dates = [1]))
p = pd.concat(dfs)
#d is output from all .txt files, so cannot be looped in cycle for
d = p.reset_index(['Date', 'Ticker', 'Close'])
d = d[(d['Ticker'].isin(ticker)) & (d['Date'] > start_date) & (d['Date'] < end_date)]
pivoted = d.pivot_table(index = ['Date'], columns =['Ticker'])
pivoted.columns = pivoted.columns.droplevel(0)
return pivoted
#function isin need list of columns, so 'or' can be replaced by ','
#arguments are changed for testing: 'HDFC' to 'AGCNET' and end_date '2014-12-31' to '2015-12-31'
closes = get_historical_closes(['LT','AGCNET','ACC'], '1999-01-01', '2015-12-31')
print closes
Related
Split a CSV in three parts and calculating the mean
I have a file containing:
Time 60Ni 61Ni 62Ni 63Cu 64Ni 65Cu 66Zn
0. 9.13242244720459 0.406570166349411 1.326429009437561 5.754200458526611 0.4233334958553314 2.68562912940979 4.148788005113602e-002
8.390999794006348 9.187464714050293 0.4089393615722656 1.334462523460388 5.790649890899658 0.425884485244751 2.702604055404663 4.17313240468502e-002
16.78300094604492 9.254316329956055 0.4119723737239838 1.344084143638611 5.832504749298096 0.428943395614624 2.722275018692017 4.203101620078087e-002
25.17399978637695 9.19857120513916 0.4094997346401215 1.336091756820679 5.791898727416992 0.4264563024044037 2.703336715698242 4.185733571648598e-002
33.56499862670898 9.194388389587402 0.4092871248722076 1.335391044616699 5.794968605041504 0.4264419078826904 2.704529047012329 4.192239791154862e-002
41.95600128173828 9.162041664123535 0.4078944325447083 1.330722570419312 5.766440868377686 0.425002932548523 2.691519498825073 4.182799160480499e-002
50.34700012207031 9.190646171569824 0.4091125726699829 1.334963202476502 5.786285877227783 0.426413893699646 2.700882434844971 4.196327552199364e-002
58.73799896240234 9.211565971374512 0.4100649058818817 1.337916374206543 5.8003830909729 0.4273969829082489 2.707314252853394 4.207673668861389e-002
67.12799835205078 9.240947723388672 0.4113766849040985 1.342136979103088 5.822870254516602 0.4287911653518677 2.717630624771118 4.222121462225914e-002
75.51899719238281 9.208130836486816 0.4099342525005341 1.337505698204041 5.802256584167481 0.4273860156536102 2.708084583282471 4.214133694767952e-002
83.91000366210938 9.196262359619141 0.4093911945819855 1.335786700248718 5.799176692962647 0.4268693923950195 2.706451416015625 4.215647280216217e-002
92.30100250244141 9.213265419006348 0.4101545214653015 1.338128447532654 5.807514190673828 0.4277283549308777 2.71068549156189 4.221603646874428e-002
100.6920013427734 9.163029670715332 0.407885879278183 1.330831050872803 5.775251865386963 0.4254410266876221 2.695534229278565 4.204751178622246e-002
109.0839996337891 9.144490242004395 0.4070722758769989 1.328153848648071 5.764679908752441 0.4246650040149689 2.690402746200562 4.198652133345604e-002
117.4749984741211 9.114171028137207 0.4057718515396118 1.32369875907898 5.745044231414795 0.4233448505401611 2.681406497955322 4.190905019640923e-002
125.8659973144531 9.149589538574219 0.407274603843689 1.328810453414917 5.766050815582275 0.4248199760913849 2.691139459609985 4.200970754027367e-002
134.2570037841797 9.168668746948242 0.4081465899944305 1.331702351570129 5.777794361114502 0.4256783723831177 2.696741819381714 4.206346347928047e-002
142.6479949951172 9.11380672454834 0.4057287871837616 1.323864817619324 5.740524291992188 0.4232001006603241 2.67945122718811 4.187140986323357e-002
151.0390014648438 9.100893974304199 0.4051263332366943 1.321851253509522 5.729655265808106 0.4226666390895844 2.674278259277344 4.182597994804382e-002
159.4299926757813 9.072731971740723 0.4039073586463928 1.317763328552246 5.713830471038818 0.4213792979717255 2.666974782943726 4.169051349163055e-002
167.8209991455078 9.186164855957031 0.4089057147502899 1.334116697311401 5.786634922027588 0.4264728426933289 2.700879812240601 4.211126267910004e-002
176.2129974365234 9.13982105255127 0.4068569839000702 1.327479124069214 5.76115083694458 0.4244593381881714 2.688895463943481 4.199059307575226e-002
184.60400390625 9.146007537841797 0.4071221053600311 1.328468441963196 5.762693881988525 0.4247534275054932 2.689634084701538 4.1985172778368e-002
192.9949951171875 9.18150806427002 0.4086942672729492 1.333438873291016 5.785679817199707 0.4262394905090332 2.700178623199463 4.207265004515648e-002
201.3860015869141 9.134004592895508 0.4066038727760315 1.326677560806274 5.753909587860107 0.424109697341919 2.685543775558472 4.191514849662781e-002
209.7769927978516 9.192599296569824 0.4091922044754028 1.335113883018494 5.792657852172852 0.4266164898872376 2.703598737716675 4.208896681666374e-002
218.1679992675781 9.166966438293457 0.4080702364444733 1.331447958946228 5.776984214782715 0.4254603683948517 2.696239709854126 4.19912114739418e-002
226.5590057373047 9.166423797607422 0.4080766439437866 1.331416010856628 5.771696090698242 0.4254250526428223 2.693812847137451 4.191195592284203e-002
234.9510040283203 9.122139930725098 0.4060815274715424 1.325031995773315 5.74381160736084 0.4234589040279388 2.680959224700928 4.174426198005676e-002
243.3419952392578 9.178729057312012 0.4085982143878937 1.333097338676453 5.783432006835938 0.4259471595287323 2.699411153793335 4.196531698107719e-002
251.7330017089844 9.196023941040039 0.4093179702758789 1.335668444633484 5.792133331298828 0.4266210496425629 2.703416347503662 4.196692258119583e-002
260.1239929199219 9.195613861083984 0.4093446731567383 1.33561098575592 5.790852546691895 0.4264806509017944 2.702755451202393 4.19374406337738e-002
268.5150146484375 9.124658584594727 0.4061901867389679 1.325218439102173 5.749895572662354 0.4233379364013672 2.683579206466675 4.166891798377037e-002
276.906005859375 9.071592330932617 0.4038631021976471 1.317633748054504 5.711780071258545 0.4209088683128357 2.666091680526733 4.146279022097588e-002
285.2969970703125 9.090703010559082 0.4047099351882935 1.320350289344788 5.724553108215332 0.4218063056468964 2.671880960464478 4.148663952946663e-002
293.68798828125 9.049410820007324 0.4028385281562805 1.314435601234436 5.699662208557129 0.4198987782001495 2.660340070724487 4.135752841830254e-002
302.0790100097656 9.158493995666504 0.4077092707157135 1.330130934715271 5.770212650299072 0.4247544705867767 2.693133354187012 4.172087088227272e-002
310.4700012207031 9.294267654418945 0.4137440025806427 1.350019454956055 5.85582971572876 0.4307662844657898 2.733232498168945 4.217509180307388e-002
318.8609924316406 9.266000747680664 0.4124558866024017 1.34581983089447 5.838682651519775 0.429353654384613 2.724989175796509 4.206011816859245e-002
327.2520141601563 9.227903366088867 0.4107420146465302 1.340180039405823 5.813295841217041 0.4277106523513794 2.713207006454468 4.191378504037857e-002
335.6430053710938 9.248990058898926 0.4117128551006317 1.343235015869141 5.836093425750732 0.4286618232727051 2.72357988357544 4.200825467705727e-002
344.0339965820313 9.200018882751465 0.4095089137554169 1.336208343505859 5.805673122406006 0.4264824092388153 2.709526300430298 4.185647144913673e-002
352.4259948730469 9.162602424621582 0.4079090356826782 1.330750703811646 5.780079364776611 0.4248281121253967 2.697546243667603 4.17003221809864e-002
360.8169860839844 9.165441513061523 0.4079831540584564 1.331099987030029 5.780121326446533 0.424967348575592 2.697607517242432 4.169800505042076e-002
369.2070007324219 9.242767333984375 0.4114582240581513 1.342459917068481 5.828019142150879 0.4283893704414368 2.719994068145752 4.194791615009308e-002
377.5989990234375 9.211434364318848 0.4100139439105988 1.337894320487976 5.801908493041992 0.4268820583820343 2.708046913146973 4.185103997588158e-002
385.989990234375 9.168110847473145 0.4081266224384308 1.33171010017395 5.772421360015869 0.4250668585300446 2.694308280944824 4.166359454393387e-002
394.3810119628906 9.162002563476563 0.4078731238842011 1.330778479576111 5.770648956298828 0.4247135519981384 2.693532466888428 4.165602847933769e-002
402.7720031738281 9.219051361083984 0.4104039072990418 1.339054584503174 5.805272579193115 0.4273586571216583 2.709418296813965 4.186749085783958e-002
411.1640014648438 9.225748062133789 0.4106448590755463 1.340008854866028 5.808595180511475 0.4276045560836792 2.711185216903687 4.189140349626541e-002
425.0020141601563 9.11283016204834 0.4056265950202942 1.323553919792175 5.742629528045654 0.4226277768611908 2.680011749267578 4.150775447487831e-002
433.3930053710938 9.15496826171875 0.4075464010238648 1.329663395881653 5.76693058013916 0.4244976043701172 2.691663980484009 4.165017232298851e-002
441.7839965820313 9.179342269897461 0.4086317718029022 1.333258748054504 5.783347606658936 0.4256252646446228 2.699387073516846 4.177364706993103e-002
450.1759948730469 9.202337265014648 0.4096647799015045 1.336641907691956 5.799064636230469 0.4267286956310272 2.706497669219971 4.189135506749153e-002
458.5669860839844 9.126877784729004 0.4062632024288178 1.325594425201416 5.7450852394104 0.4234336316585541 2.681554317474365 4.164514690637589e-002
466.9580078125 9.130221366882324 0.4063588082790375 1.326080322265625 5.750959873199463 0.4235436022281647 2.6843581199646 4.169851914048195e-002
475.3489990234375 9.142138481140137 0.4069503247737885 1.32788360118866 5.753814697265625 0.4240946471691132 2.685687065124512 4.17218841612339e-002
483.739990234375 9.144487380981445 0.4070816040039063 1.328163623809815 5.764283180236816 0.4243338704109192 2.69016432762146 4.180238768458366e-002
492.1310119628906 9.213832855224609 0.4101627767086029 1.338177442550659 5.806262969970703 0.4273685812950134 2.709989309310913 4.204079136252403e-002
500.5220031738281 9.151962280273438 0.4073929488658905 1.329235196113586 5.765473365783691 0.4247141480445862 2.691080808639526 4.187702387571335e-002
508.9129943847656 9.133262634277344 0.4065472185611725 1.326548576354981 5.755089282989502 0.4239353835582733 2.685916900634766 4.184074699878693e-002
517.3040161132813 9.194231033325195 0.4092318415641785 1.335361480712891 5.791540622711182 0.4266365468502045 2.703181505203247 4.204431921243668e-002
525.6950073242188 9.174141883850098 0.4084053635597229 1.332433700561523 5.780707836151123 0.4258663356304169 2.697983264923096 4.203671962022781e-002
534.0869750976563 9.127938270568848 0.4063973724842072 1.325674772262573 5.753820896148682 0.4238673448562622 2.685414791107178 4.189241677522659e-002
542.4769897460938 9.228574752807617 0.4108735322952271 1.340509295463562 5.816771030426025 0.4283493161201477 2.714869976043701 4.227539896965027e-002
550.8679809570313 9.247261047363281 0.4116438031196594 1.34306275844574 5.829936504364014 0.4292499721050263 2.720824480056763 4.234698414802551e-002
559.2589721679688 9.259587287902832 0.4121484756469727 1.344773530960083 5.840207099914551 0.4296930134296417 2.725474834442139 4.239725694060326e-002
567.6500244140625 9.236879348754883 0.4112152457237244 1.341552734375 5.824738502502441 0.4288162887096405 2.718418121337891 4.232741147279739e-002
576.041015625 9.265199661254883 0.4123806655406952 1.345624566078186 5.837865352630615 0.4300332069396973 2.724727630615234 4.243086278438568e-002
584.4310302734375 9.193467140197754 0.4092609882354736 1.335316061973572 5.791056632995606 0.4267773926258087 2.702801465988159 4.214197397232056e-002
592.822021484375 9.178906440734863 0.408621221780777 1.333141565322876 5.783803462982178 0.4262367188930512 2.699366569519043 4.21367958188057e-002
601.2139892578125 9.179999351501465 0.4086976051330566 1.333412766456604 5.781562805175781 0.4262183606624603 2.698424100875855 4.212524741888046e-002
609.60498046875 9.158502578735352 0.4077076315879822 1.330240249633789 5.771774768829346 0.4252981841564179 2.693920612335205 4.206201061606407e-002
617.9949951171875 9.168906211853027 0.4081432521343231 1.331776857376099 5.777164459228516 0.4257596433162689 2.696363210678101 4.212769865989685e-002
626.385986328125 9.148199081420898 0.4072228968143463 1.328739166259766 5.764687061309815 0.4248482882976532 2.690601110458374 4.204926639795303e-002
634.7769775390625 9.153997421264648 0.4075290560722351 1.329600691795349 5.76605749130249 0.4250805974006653 2.691195011138916 4.203818738460541e-002
643.1680297851563 9.142102241516113 0.4070025384426117 1.327812790870667 5.758194923400879 0.4244733154773712 2.687539577484131 4.197685047984123e-002
651.5599975585938 9.157526016235352 0.4076575040817261 1.33014190196991 5.771289825439453 0.4252424538135529 2.693483829498291 4.207025840878487e-002
659.9509887695313 9.142055511474609 0.4069408476352692 1.327834606170654 5.75890064239502 0.4245132505893707 2.687950849533081 4.196911677718163e-002
668.3410034179688 9.163941383361816 0.4079061448574066 1.331052899360657 5.773416519165039 0.425525963306427 2.694749593734741 4.208214208483696e-002
676.7329711914063 9.214210510253906 0.4101268947124481 1.338269472122192 5.804011821746826 0.4277287721633911 2.70874834060669 4.224084317684174e-002
685.1240234375 9.221725463867188 0.410546600818634 1.33942449092865 5.808478832244873 0.4280569553375244 2.710729837417603 4.224072396755219e-002
693.5139770507813 9.195225715637207 0.4093619287014008 1.335615515708923 5.792295932769775 0.4269255101680756 2.703481912612915 4.215554893016815e-002
701.905029296875 9.236662864685059 0.4111031889915466 1.341474533081055 5.820279121398926 0.4286713898181915 2.716408491134644 4.231745004653931e-002
710.2969970703125 9.219303131103516 0.4103749394416809 1.33903431892395 5.809108257293701 0.4279004633426666 2.711240530014038 4.220414161682129e-002
718.68798828125 9.196757316589356 0.4093507528305054 1.335767865180969 5.794125556945801 0.4269102811813355 2.704240798950195 4.217429086565971e-002
727.0789794921875 9.169294357299805 0.4081831276416779 1.331677913665772 5.778267860412598 0.4257012009620667 2.696781396865845 4.20493595302105e-002
735.468994140625 9.254044532775879 0.4119507372379303 1.344122529029846 5.83418083190918 0.4294586181640625 2.722884654998779 4.238997399806976e-002
743.8610229492188 9.224509239196777 0.4105926156044006 1.339867234230042 5.812450408935547 0.4280983507633209 2.712637424468994 4.227783530950546e-002
752.2520141601563 9.167038917541504 0.4080414175987244 1.331365466117859 5.778883457183838 0.4256396591663361 2.697120428085327 4.206839948892593e-002
760.6430053710938 9.156136512756348 0.407585471868515 1.329828977584839 5.771244049072266 0.4251766502857208 2.693709135055542 4.204395413398743e-002
769.0339965820313 9.206752777099609 0.4098866879940033 1.337259769439697 5.798995018005371 0.4273804128170013 2.706660270690918 4.218916967511177e-002
777.4249877929688 9.185664176940918 0.4088890254497528 1.33407187461853 5.787529468536377 0.426471084356308 2.701387643814087 4.21074777841568e-002
785.8159790039063 9.148477554321289 0.4072705209255219 1.328797459602356 5.764423847198486 0.4247606992721558 2.690322160720825 4.200183600187302e-002
794.2069702148438 9.139849662780762 0.4068310558795929 1.327486157417297 5.760977268218994 0.4244396984577179 2.688838005065918 4.198827594518662e-002
802.5980224609375 9.198716163635254 0.409488320350647 1.336077690124512 5.797767639160156 0.4270517528057098 2.705855131149292 4.215721413493156e-002
810.989013671875 9.175697326660156 0.4084174335002899 1.332631826400757 5.781099796295166 0.425992488861084 2.698201894760132 4.206936806440353e-002
819.3800048828125 9.106189727783203 0.4053537547588348 1.322664737701416 5.740387916564941 0.4229016602039337 2.679165840148926 4.18708510696888e-002
827.77099609375 9.11962890625 0.4059470593929291 1.324671149253845 5.745753765106201 0.4235488474369049 2.681836843490601 4.189123585820198e-002
836.1619873046875 9.221225738525391 0.4104022979736328 1.33923864364624 5.813970565795898 0.4279847741127014 2.713436365127564 4.224034398794174e-002
849.9970092773438 9.109155654907227 0.4055195748806 1.323018074035645 5.738785743713379 0.4229097962379456 2.678738832473755 4.17560487985611e-002
858.3880004882813 9.081585884094238 0.4043126106262207 1.319140315055847 5.720804691314697 0.4216950535774231 2.670202732086182 4.168836399912834e-002
866.7789916992188 9.1737060546875 0.4083895683288574 1.332486510276794 5.779799461364746 0.4258598983287811 2.697497129440308 4.201843962073326e-002
875.1699829101563 9.215715408325195 0.4102407991886139 1.33849024772644 5.806502342224121 0.4276199042797089 2.710031509399414 4.214433580636978e-002
883.5609741210938 9.29750919342041 0.4138506650924683 1.350215315818787 5.858696460723877 0.4313125610351563 2.734477758407593 4.240995645523071e-002
891.9520263671875 9.251111030578613 0.411830872297287 1.343641996383667 5.826048374176025 0.4292575418949127 2.719125270843506 4.226363822817802e-002
900.343017578125 9.236968994140625 0.411191999912262 1.341637492179871 5.816394329071045 0.4285323023796082 2.71470046043396 4.218020662665367e-002
908.7340087890625 9.18012809753418 0.4086549580097199 1.333361864089966 5.780932903289795 0.4260410964488983 2.698340177536011 4.198113456368446e-002
917.125 9.18910026550293 0.4090204238891602 1.334587931632996 5.791236877441406 0.426427572965622 2.702847242355347 4.205641150474548e-002
925.5159912109375 9.163248062133789 0.4078385829925537 1.330891489982605 5.775006771087647 0.4252764880657196 2.695378065109253 4.195348545908928e-002
933.906982421875 9.184928894042969 0.4089162349700928 1.334069848060608 5.789799213409424 0.42618727684021 2.702196598052979 4.199947416782379e-002
942.2979736328125 9.157343864440918 0.4076671004295349 1.330055475234985 5.770273208618164 0.4249707460403442 2.693178653717041 4.188660532236099e-002
950.6890258789063 9.162631988525391 0.4078827202320099 1.330793499946594 5.77417516708374 0.4251722097396851 2.695005416870117 4.190302640199661e-002
959.0800170898438 9.114273071289063 0.4057436585426331 1.323749780654907 5.743786811828613 0.4230408370494843 2.680756568908691 4.173881560564041e-002
967.4710083007813 9.244811058044434 0.4115355014801025 1.34266197681427 5.823981761932373 0.4288525879383087 2.718071460723877 4.214448481798172e-002
975.8619995117188 9.219685554504395 0.4104566872119904 1.339130640029907 5.808487892150879 0.4276332259178162 2.710957288742065 4.206658154726028e-002
984.2529907226563 9.184207916259766 0.4088565707206726 1.33392071723938 5.792478561401367 0.4260831475257874 2.703508853912354 4.195259138941765e-002
992.6439819335938 9.13871955871582 0.4068254828453064 1.327333569526672 5.761001586914063 0.4240987598896027 2.688708066940308 4.179005324840546e-002
1001.034973144531 9.151439666748047 0.4073895514011383 1.329284429550171 5.767615795135498 0.4246693849563599 2.691930532455444 4.182363301515579e-002
1009.424987792969 9.19940185546875 0.409492164850235 1.335996866226196 5.800271034240723 0.4267957508563995 2.70706057548523 4.198677837848663e-002
1017.815979003906 9.255974769592285 0.4120437800884247 1.344139099121094 5.840244770050049 0.4293366670608521 2.725528001785278 4.220050573348999e-002
1026.20703125 9.220073699951172 0.4104630351066589 1.339051723480225 5.81441593170166 0.4276903867721558 2.713610172271729 4.208677262067795e-002
1034.598022460938 9.158895492553711 0.4077011644840241 1.330096125602722 5.776969432830811 0.4249850511550903 2.696006536483765 4.186514392495155e-002
1042.989013671875 9.135567665100098 0.4066715240478516 1.326890826225281 5.756415843963623 0.423865556716919 2.686625719070435 4.174899682402611e-002
1051.380981445313 9.150594711303711 0.4073532521724701 1.329049825668335 5.765689849853516 0.4245824813842773 2.691075325012207 4.179978370666504e-002
1059.77197265625 9.146571159362793 0.4071609079837799 1.32847785949707 5.760791778564453 0.4242803156375885 2.688825607299805 4.17768582701683e-002
1068.162963867188 9.131063461303711 0.4064978063106537 1.326229453086853 5.752644538879395 0.4236991405487061 2.684972286224365 4.172741994261742e-002
1076.553955078125 9.098221778869629 0.4049918949604034 1.321496725082398 5.731342792510986 0.4222320318222046 2.675036668777466 4.162869602441788e-002
1084.944946289063 9.169441223144531 0.4081719219684601 1.331780910491943 5.776838779449463 0.4254011511802673 2.696260452270508 4.184866324067116e-002
1093.337036132813 9.187003135681152 0.4089777171611786 1.334323048591614 5.790809154510498 0.4261792898178101 2.702747344970703 4.196572676301003e-002
1101.72802734375 9.179986953735352 0.4086208045482636 1.333386778831482 5.783829689025879 0.4258585274219513 2.699674844741821 4.191147163510323e-002
1110.119018554688 9.200528144836426 0.4095506370067596 1.336296439170837 5.797418117523193 0.4267379641532898 2.7057945728302 4.19546514749527e-002
1118.509033203125 9.158334732055664 0.4076752066612244 1.330214262008667 5.770383834838867 0.4248470067977905 2.693165063858032 4.180992022156715e-002
1126.900024414063 9.194581985473633 0.4093466997146606 1.335410833358765 5.798298358917236 0.4264914393424988 2.706053495407105 4.194727912545204e-002
1135.291015625 9.176510810852051 0.4084961414337158 1.3328697681427 5.778421401977539 0.4256733357906342 2.697108507156372 4.18514646589756e-002
1143.682983398438 9.163573265075684 0.4079014360904694 1.330968260765076 5.773004055023193 0.4250616133213043 2.694518804550171 4.183558747172356e-002
1152.072998046875 9.159396171569824 0.4077317416667938 1.330322265625 5.771379947662354 0.4248954653739929 2.693806171417236 4.181275144219399e-002
1160.464965820313 9.165866851806641 0.4080128371715546 1.331347465515137 5.772171497344971 0.4252021610736847 2.694234848022461 4.181317612528801e-002
1168.85595703125 9.151269912719727 0.407374233007431 1.329119086265564 5.760807991027832 0.424500435590744 2.688781023025513 4.176882281899452e-002
1177.246948242188 9.141792297363281 0.4069608747959137 1.327713966369629 5.75624418258667 0.4241056740283966 2.68661379814148 4.173726961016655e-002
1185.636962890625 9.130838394165039 0.406494677066803 1.326230525970459 5.751668930053711 0.4236221015453339 2.684362649917603 4.168353974819183e-002
1194.027954101563 9.206241607666016 0.4098086059093475 1.337079763412476 5.802299022674561 0.4269396662712097 2.707928895950317 4.194400832056999e-002
1202.4189453125 9.17149543762207 0.4083086550235748 1.332085609436035 5.776546001434326 0.4253532886505127 2.696049451828003 4.180750250816345e-002
1210.81005859375 9.140050888061523 0.4068616330623627 1.327504873275757 5.760209083557129 0.4239790141582489 2.6883225440979 4.170787334442139e-002
1219.201049804688 9.165439605712891 0.4079880714416504 1.331203103065491 5.77871561050415 0.4250532984733582 2.697003841400147 4.180311039090157e-002
1227.593017578125 9.177500724792481 0.4085498750209808 1.332932233810425 5.783236026763916 0.4255987405776978 2.699163913726807 4.181493073701859e-002
1235.984008789063 9.177756309509277 0.408606618642807 1.33305811882019 5.782862663269043 0.4256067276000977 2.699074268341065 4.182154312729836e-002
1244.375 9.143049240112305 0.4070280194282532 1.327925682067871 5.766200542449951 0.4240804016590118 2.691066265106201 4.171686246991158e-002
1252.765991210938 9.110544204711914 0.4055243730545044 1.323151469230652 5.742761135101318 0.422651082277298 2.680213212966919 4.159015789628029e-002
1261.156982421875 9.153350830078125 0.4074757993221283 1.329340934753418 5.772144794464111 0.4244934320449829 2.693885564804077 4.173129424452782e-002
I want to split the file in three parts á 50 rows:
data = pd.read_csv(file, sep='\t', names=['Time', '60Ni', '61Ni', '62Ni', '63Cu', '64Ni', '65Cu', '66Zn'], skiprows=3, nrows=50, index_col=False, dtype=float)
data2 = pd.read_csv(file, sep='\t', names=['Time', '60Ni', '61Ni', '62Ni', '63Cu', '64Ni', '65Cu', '66Zn'], skiprows=53, nrows=50, index_col=False, dtype=float)
data3 = pd.read_csv(file, sep='\t', names=['Time', '60Ni', '61Ni', '62Ni', '63Cu', '64Ni', '65Cu', '66Zn'], skiprows=103, nrows=50, index_col=False, dtype=float)
Then I'm removing outliers with:
cols = list(data.drop(columns='Time').columns)
datao = pd.DataFrame({'Time':data['Time']})
datao[cols] = data[cols].where(np.abs(stats.zscore(data[cols])) < 2)
cols = list(data2.drop(columns='Time').columns)
data2o = pd.DataFrame({'Time':data2['Time']})
data2o[cols] = data2[cols].where(np.abs(stats.zscore(data2[cols])) < 2)
data2o[cols] = data2o[cols].mean()
cols = list(data3.drop(columns='Time').columns)
data3o = pd.DataFrame({'Time':data3['Time']})
data3o[cols] = data3[cols].where(np.abs(stats.zscore(data3[cols])) < 2)
data3o[cols] = data3o[cols].mean()
Does this make sense so far?
And now I would like to create a mean of datao, data2o and data3o seperately, resulting in three values for 60Ni, 61Ni, 62Ni, 63Cu, 64Ni, 65Cu, 66Zn. After that, I want to make a mean of these three values again. How should I do this?
I tried to make it this way:
mean_filtered_transposed = pd.DataFrame(data=np.mean(data)).T
mean_filtered_transposed['Time'] = pd.to_datetime(mean_filtered_transposed["Time"], unit='s')mean_filtered_transposed = pd.DataFrame(data=np.mean(data)).T
mean_filtered_transposed['Time'] = pd.to_datetime(mean_filtered_transposed["Time"], unit='s')
mean_filtered_transposed2 = pd.DataFrame(data=np.mean(data2)).T
mean_filtered_transposed2['Time'] = pd.to_datetime(mean_filtered_transposed["Time"], unit='s')
mean_filtered_transposed3 = pd.DataFrame(data=np.mean(data3)).T
mean_filtered_transposed3['Time'] = pd.to_datetime(mean_filtered_transposed3["Time"], unit='s')
mean_all = pd.concat(mean_filtered_transposed, mean_filtered_transposed2, mean_filtered_transposed3)
However, this results in:
"TypeError: first argument must be an iterable of pandas objects, you passed an object of type "DataFrame""
Based on documentation: objs: a sequence or mapping of Series or DataFrame objects So: s1 = pd.Series(['a', 'b']) s2 = pd.Series(['c', 'd']) pd.concat([s1, s2]) result: But: s1 = pd.Series(['a', 'b']) s2 = pd.Series(['c', 'd']) pd.concat(s1, s2) generates:
Pandas - EmptyDataError: No columns to parse from file when reading stock .csv file
Let me first start by saying I have gone through and done my due diligence trying to find a solution based on questions previously asked on the web.
I've run into an odd bug in my code that I really cannot explain...
So far my code executes the following:
take stock symbols and write OHLC data to a CSV file
loop through the directory that contains the CSV files and use that data to calculate technical indicators
add the technical indicator data to the same CSV file
So the bug is that it executes everything perfectly (99 stocks) EXCEPT for ZM.csv (Zoom). The error that it prints is"
pandas.errors.EmptyDataError: No columns to parse from file.
So to troubleshoot I copied and pasted the data from ZM.csv into a CSV that I know ran fine (I used AAPL) and it actually executed fine. Next, I took the working data from AAPL.csv, pasted it into ZM.csv and ran it again. It throws the same error. I also tried renaming the file to ZMI (randomly) and it worked.
This led me to believe that for some unknown reason that the FILENAME is the root issue. The part where I first create the CSV files, I changed the name of the file to be {symbol}1.csv, {symbol}_.csv, and {symbol}I.csv to no avail. Lastly, I combined the two files together and did not mess with anything else. It worked. Does anyone know why?
The flow is to first run bars.py, check the data/ohlc/ directory CSV files (should only have the OHLC data), run technical_analysis.py, and then check the CSV files again (now with technical indicators).
[bar.py]
from config import *
from datetime import datetime
import requests, json
holdings = open('data/qqq.csv').readlines()
symbols_list = [holding.split(',')[2].strip() for holding in holdings][1:]
symbols = ','.join(symbols_list)
minute_bars_url = '{}/1Min?symbols={}&limit=100'.format(BARS_URL, symbols)
r = requests.get(minute_bars_url, headers=HEADERS)
ohlc_data = r.json()
for symbol in ohlc_data:
filename = 'data/ohlc/{}.csv'.format(symbol)
f = open(filename, 'w+')
f.write('Timestamp,Open,High,Low,Close,Volume\n')
for bar in ohlc_data[symbol]:
t = datetime.fromtimestamp(bar['t'])
timestamp = t.strftime('%I:%M:%S%p-%Z%Y-%m-%d')
line = '{},{},{},{},{},{}\n'.format(timestamp, bar['o'], bar['h'],
bar['l'], bar['c'], bar['v'])
f.write(line)
The variables symbols_list and symbols print as follows:
symbols_list = ['AAPL', 'MSFT', 'AMZN', 'FB', 'GOOGL', 'GOOG', 'TSLA', 'NVDA', 'PYPL', 'ADBE', 'INTC', 'NFLX', 'CMCSA', 'PEP', 'COST', 'CSCO', 'AVGO', 'QCOM', 'TMUS', 'AMGN', 'TXN', 'CHTR', 'SBUX', 'ZM', 'AMD', 'INTU', 'ISRG', 'MDLZ', 'JD', 'GILD', 'BKNGLD', 'BKNG', 'FISV', 'MELI', 'ATVI', 'ADP', 'CSX', 'REGN', 'MU', 'AMAT', 'ADSK', 'VRTX', 'LRCX', 'ILMN', 'ADI', 'BIIB', 'MNST', 'EXC', 'KDP', 'LULU', 'DOCU', 'WDAY', 'CTSH', 'KHC', 'NXPI', 'BIDU', 'XEL', 'DXCM', 'EBAY', 'EA', 'ID', 'SNPS',XX', 'CTAS', 'SNPS', 'ORLY', 'SGEN', 'SPLK', 'ROST', 'WBA', 'KLAC', 'NTES', 'PCAR', 'CDNS', 'MAR', 'VRSK', 'PAYX', 'ASML', 'ANSS', 'MCHP', 'XLNX', 'MRNA', 'CPRT', 'ALGN', 'PDD', 'ALXN', 'SIRI', 'FAST', 'SWKS', 'VRSN', 'DLTR', 'CE 'TTWO', 'RN', 'MXIM', 'INCY', 'TTWO', 'CDW', 'CHKP', 'CTXS', 'TCOM', 'BMRN', 'ULTA', 'EXPE', 'FOXA', 'LBTYK', 'FOX', 'LBTYA']
symbols = AAPL,MSFT,AMZN,FB,GOOGL,GOOG,TSLA,NVDA,PYPL,ADBE,INTC,NFLX,CMCSA,PEP,COST,CSCO,AVGO,QCOM,TMUS,AMGN,TXN,CHTR,SBUX,ZM,AMD,INTU,ISRG,MDLZ,JD,GILD,BKNG,FISV,MELI,ATVI,ADP,CSX,REGN,MU,AMAT,ADSK,VRTX,LRCX,ILMN,ADI,BIIB,MNST,EXC,KDP,LULU,DOCU,WDAU,DOCU,WDAY,CTSH,KHC,NXPI,BIDU,XEL,DXCM,EBAY,EA,IDXX,CTAS,SNPS,ORLY,SGEN,SPLK,ROST,WBA,KLAC,NTES,PCAR,CDNS,MAR,VRSK,PAYX,ASML,ANSS,MCHP,XLNX,MRNA,CPRT,ALGN,PDD,ALXN,SIRI,FAST,SWKS,VRSN,DLTR,CERN,MXIM,INCY,TTWO,CDW,CHKP,CTXS,TCOM,EXPE,FOXA,BMRN,ULTA,EXPE,FOXA,LBTYK,FOX,LBTYA
So ZM is not listed last.
[technical_analysis.py]
import btalib
import pandas as pd
from datetime import datetime
from bars import ohlc_data
from bars import symbols_list as symbols
for symbol in symbols:
try:
file_path = f'data/ohlc/{symbol}.csv'
dataframe = pd.read_csv(file_path,
parse_dates=True,
index_col='Timestamp')
sma6 = btalib.sma(dataframe, period=6)
sma10 = btalib.sma(dataframe, period=10)
rsi = btalib.rsi(dataframe)
macd = btalib.macd(dataframe)
dataframe['SMA-6'] = sma6.df
dataframe['SMA-10'] = sma10.df
dataframe['RSI'] = rsi.df
dataframe['MACD'] = macd.df['macd']
dataframe['Signal'] = macd.df['signal']
dataframe['Histogram'] = macd.df['histogram']
f = open(file_path, 'w+')
dataframe.to_csv(file_path, sep=',', index=True)
except:
print(f'{symbol} is not writing the technical data.')
I think the error might be since 'ZM' is the last symbol in holdings, it contains some whitespace, due to in [bar.py] you created holdings the following way (instead of just the normal pd.read_csv):
holdings = open('data/qqq.csv').readlines()
symbols_list = [holding.split(',')[2].strip() for holding in holdings][1:]
symbols = ','.join(symbols_list)
You can probably reduce the code more to get a minimally viable example. I suspect there is something funny in the qqq.csv file and the split/strip code that makes the last entry not quite what you want.
Hopefully, that'll be clear printing the variable values as below.
with data/qqq.csv like
xname,yname,symbol
xxx,yyy,ZM
and py example
def write_OHLC(fname):
"write example data to a file"
f = open(fname, 'w+')
f.write('Timestamp,Open,High,Low,Close,Volume\n')
# IRL, would parse json and spitout meaningful values
f.write('2020-10-13 16:30,1,10,5,100\n')
def all_symbols():
"get list of all symbols from qqq.csv"
holdings = open('data/qqq.csv').readlines()
symbols_list = [holding.split(',')[2].strip() for holding in holdings][1:]
return symbols_list
# issue saving/reading last(?) symbol
symbols = all_symbols()
print(symbols)
# check just zoom
zm_sym = symbols[-1]
fname = f'data/ohlc/{zm_sym}.csv'
# inspect
print(zm_sym)
print(fname)
# write and read back
write_OHLC(fname)
ZM = pd.read_csv(fname,
parse_dates=True,
index_col='Timestamp')
print(ZM)
How to display all the values of an index in a Series?
I don't know how to display all the indexes of a Series.
# current settings
import pandas as pd
pd.set_option('display.max_columns', 10000)
pd.set_option('display.max_rows', 10000)
s = pd.Series(np.random.randn(199), index=['place_id', 'name', 'formatted_address', 'formatted_phone_number',
'num_comments', 'rating', 'price', 'website', 'lng', 'lat', 'category',
'permanently_closed', 'lunes_open', 'lunes_close_mid', 'lunes_open_mid',
'lunes_close', 'martes_open', 'martes_close_mid', 'martes_open_mid',
'martes_close', 'miércoles_open', 'miércoles_close_mid',
'miércoles_open_mid', 'miércoles_close', 'jueves_open',
'jueves_close_mid', 'jueves_open_mid', 'jueves_close', 'viernes_open',
'viernes_close_mid', 'viernes_open_mid', 'viernes_close', 'sábado_open',
'sábado_close_mid', 'sábado_open_mid', 'sábado_close', 'domingo_open',
'domingo_close_mid', 'domingo_open_mid', 'domingo_close', 'Monday_00',
'Monday_01', 'Monday_02', 'Monday_03', 'Monday_04', 'Monday_05',
'Monday_06', 'Monday_07', 'Monday_08', 'Monday_09','Monday_10', 'Monday_11', 'Monday_12', 'Monday_13', 'Monday_14',
'Monday_15', 'Monday_16', 'Monday_17', 'Monday_18', 'Monday_19',
'Monday_20', 'Monday_21', 'Monday_22', 'Monday_23', 'Monday_peak',
'Tuesday_00', 'Tuesday_01', 'Tuesday_02', 'Tuesday_03', 'Tuesday_04',
'Tuesday_05', 'Tuesday_06', 'Tuesday_07', 'Tuesday_08', 'Tuesday_09',
'Tuesday_10', 'Tuesday_11', 'Tuesday_12', 'Tuesday_13', 'Tuesday_14',
'Tuesday_15', 'Tuesday_16', 'Tuesday_17', 'Tuesday_18', 'Tuesday_19',
'Tuesday_20', 'Tuesday_21', 'Tuesday_22', 'Tuesday_23', 'Tuesday_peak',
'Wednesday_00', 'Wednesday_01', 'Wednesday_02', 'Wednesday_03',
'Wednesday_04', 'Wednesday_05', 'Wednesday_06', 'Wednesday_07',
'Wednesday_08', 'Wednesday_09','Wednesday_11', 'Wednesday_12', 'Wednesday_13', 'Wednesday_14',
'Wednesday_15', 'Wednesday_16', 'Wednesday_17', 'Wednesday_18',
'Wednesday_19', 'Wednesday_20', 'Wednesday_21', 'Wednesday_22',
'Wednesday_23', 'Wednesday_peak', 'Thursday_00', 'Thursday_01',
'Thursday_02', 'Thursday_03', 'Thursday_04', 'Thursday_05',
'Thursday_06', 'Thursday_07', 'Thursday_08', 'Thursday_09',
'Thursday_10', 'Thursday_11', 'Thursday_12', 'Thursday_13',
'Thursday_14', 'Thursday_15', 'Thursday_16', 'Thursday_17',
'Thursday_18', 'Thursday_19', 'Thursday_20', 'Thursday_21',
'Thursday_22', 'Thursday_23', 'Thursday_peak', 'Friday_00', 'Friday_01',
'Friday_02', 'Friday_03', 'Friday_04', 'Friday_05', 'Friday_06',
'Friday_07', 'Friday_08', 'Friday_09', 'Friday_10', 'Friday_11',
'Friday_12', 'Friday_13', 'Friday_14', 'Friday_15', 'Friday_16',
'Friday_17', 'Friday_18', 'Friday_19', 'Friday_20', 'Friday_21',
'Friday_22', 'Friday_23', 'Friday_peak', 'Saturday_00', 'Saturday_01',
'Saturday_02', 'Saturday_03', 'Saturday_04', 'Saturday_05',
'Saturday_06', 'Saturday_07', 'Saturday_08', 'Saturday_09',
'Saturday_10', 'Saturday_11', 'Saturday_12', 'Saturday_13',
'Saturday_14', 'Saturday_15', 'Saturday_16', 'Saturday_17',
'Saturday_18', 'Saturday_19', 'Saturday_20', 'Saturday_21',
'Saturday_22', 'Saturday_23', 'Saturday_peak', 'Sunday_00', 'Sunday_01',
'Sunday_02', 'Sunday_03', 'Sunday_04', 'Sunday_05', 'Sunday_06',
'Sunday_07', 'Sunday_08', 'Sunday_09'])
# I ask for the index
s.index
Instead of getting all the values I only see this
Out[94]: Index(['place_id', 'name', 'formatted_address',
'formatted_phone_number',
'num_comments', 'rating', 'price', 'website', 'lng', 'lat',
...
'Sunday_00', 'Sunday_01', 'Sunday_02', 'Sunday_03', 'Sunday_04',
'Sunday_05', 'Sunday_06', 'Sunday_07', 'Sunday_08', 'Sunday_09'],
dtype='object', length=199)
Any ideas on which settings to use?
s.index.values And you can store it in a variable or print it(which seems to be the case here)
You need this: pd.options.display.max_seq_items = None
Nested dictionary groups from excel
I'm new in python and openpyxl. I started to learn in order to make my every day tasks easier and faster at my workplace.
Task:
There is an excel file with a lots of rows, looks like this
excel file
I want to create a daily report based on this excel file. In my example Today is 2019/05/08.
Expected result:
Only show the info where the date is match with Today date.
Expected structure:
required outcome
My solution
In my solution I create a list of the rows where I can find only the Today values. After that I read only that rows and create dictionaries. But the result is nothing. I also in a trouble about how to work with multiple keys. Because there are multiple issue numbers are in the list.
from datetime import datetime
import openpyxl
from openpyxl import load_workbook
from openpyxl.utils import get_column_letter
from openpyxl.utils import column_index_from_string
#Open excel file
excel_path = "\\REE.xlsx"
wb = openpyxl.load_workbook(excel_path, data_only=True)
ws_1 = wb.worksheets[1]
#The Today date. need some format due to excel date handling
today = datetime.today()
today = today.replace(hour=00, minute=00, second=00, microsecond=00)
#Crate a list of the lines where only Today values are present
issue_line_list = []
for cell in ws_1["B"]:
if cell.value == today:
issue_line = cell.row
issue_line_list.append(issue_line)
#Creare a txt file for output
file = open("daily_report.txt", "w")
#The dict what I want to use
dict = []
issue_numbers_list = []
issue = []
#Create a dict for the issues
for line in issue_line_list:
issue_number_value = ws_1.cell(row = line, column = 3).value
issue_numbers_list.append(issue_number_value)
#Create a dict for other information
for line in issue_line_list:
issue_number_value = ws_1.cell(row = line, column = 3).value
by_value = ws_1.cell(row = line, column = 2 ).value
group_value = ws_1.cell(row = line, column = 4).value
events_value = ws_1.cell(row = line, column = 5).value
deadline_value = ws_1.cell(row = line, column = 6).value
try:
deadline_value = deadline_value.strftime('%Y.%m.%d')
except:
deadline_value = ""
issue.append(issue_number_value)
issue.append(by_value)
issue.append(group_value)
issue.append(events_value)
issue.append(deadline_value)
issue.append(deadline_value)
#Append the two dict
dict.append(issue_numbers_list)
dict.append(issue)
#Save it to the txt file.
file.write(dict)
file.close()
Questions
- How to solve the multiple same key issue?
- How to create nested groups?
- What should add or delete to my code in order to get the expected result?
Remark
Openpyxl is not only option. If you have a bettwer/easier/faster way I open for every idea.
Thank you in advance for you support!
Can you try the following: import pandas as pd cols = ['date', 'by', 'issue_number', 'group', 'events', 'deadline'] req_cols = ['events', 'deadline'] data = [ ['2019-05-07', 'john', '113140', '#issue_closed', 'something different', ''], ['2019-05-08', 'david', '113140', '#task', 'something different', ''], ['2019-05-08', 'victor', '114761', '#task_result', 'something different', ''], ['2019-05-08', 'john', '114761', '#task', 'something different', '2019-05-10'], ['2019-05-08', 'david', '114761', '#task', 'something different', '2019-05-08'], ['2019-05-08', 'victor', '113140', '#task_result', 'something different', ''], ['2019-05-07', 'john', '113140', '#issue_created', 'something different', '2019-05-09'], ['2019-05-07', 'david', '113140', '#location', 'something different', ''], ['2019-05-07', 'victor', '113140', '#issue_closed', 'something different', 'done'], ['2019-05-07', 'john', '113140', '#task_result', 'something different', ''], ['2019-05-07', 'david', '113140', '#task', 'something different', '2019-05-10'], ] df = pd.DataFrame(data, columns=cols) df1 = df.groupby(['issue_number', 'group']).describe()[req_cols].droplevel(0, axis=1)['top'] df1.columns = req_cols print(df1) Output: events deadline issue_number group 113140 #issue_closed something different done #issue_created something different 2019-05-09 #location something different #task something different 2019-05-10 #task_result something different 114761 #task something different 2019-05-08 #task_result something different To open an excel file, you can do the following: df = pd.read_excel(excel_path, sheet_name=my_sheet) req_cols = ['EVENTS', 'DEADLINE'] df1 = df.groupby(['ISSUE NUMBER', 'GROUP']).describe()[req_cols].droplevel(0, axis=1)['top'] df1.columns = req_cols print(df1)
The task almost solved, but I faced a new issue.
The code:
excel_path = "\\REE.xlsx"
my_sheet = 'Events'
cols = ['DATE', 'BY', 'ISSUE NUMBER', 'GROUP', 'EVENTS', 'DEADLINE']
req_cols = ['EVENTS', 'DEADLINE']
df = pd.read_excel(excel_path, sheet_name = my_sheet, columns=cols)
today = datetime.today().strftime('%Y-%m-%d')
today_filter = (df[(df['DATE'] == today)])
df = pd.DataFrame(today_filter, columns=cols)
df1 = df.groupby(['ISSUE NUMBER', 'GROUP']).describe()[req_cols].droplevel(0, axis=1['top']
df1.columns = req_cols
print(df1)
On the 'BY' column there are same values. eg. '#task'. But the script print only once.
int his case
Required result:
114761
#task Jane another words 2019-05-10
#task result John something
#task John something else 2019-05-08
...
...
...
...
My code result:
114761
#task Jane another words 2019-05-10
#task result John something
...
...
...
John #task something else 2019-05-08 do not print it out. Why?
And there is a some result in other options also. If there are more some values at'BY' column the script print out only the first and skip the rest.
How does Python convert date value from excel
I am reading a csv file with a CDATE column. The structure of the column is:
|CDATE |
|08/28/2018|
|08/28/2018|
|08/29/2018|
|08/30/2018|
|09/02/2018|
|09/04/2018|
...
|04/10/2019|
As you can see there is duplicate date as well as missing dates in this column, and I would like to find the missing dates and add them to my dataframe.
My code is:
import matplotlib.pyplot as plt
warnings.filterwarnings("ignore")
plt.style.use('fivethirtyeight')
import pandas as pd
df = pd.read_csv("XXX.csv")
dateCol = df['CDATE'].values.tolist()
dates = pd.to_datetime(dateCol, format='%m/%d/%Y')
startDate = dates.min()
endDate = dates.max()
df = df.sort_values('CDATE')
df_plastic = df['PLASTIC'].unique()
dateRange = pd.date_range(startDate, endDate)
df_date = df['CDATE'].unique()
for cursorDate in dateRange:
if (cursorDate in df_date) is False:
print('Data is missing date {} from range {}'.format(cursorDate, df_date))
But the output is:
Data is missing date 2019-02-21 00:00:00 from ['01/01/2019' '01/02/2019' '01/03/2019' '01/04/2019' '01/05/2019'
'01/07/2019' '01/08/2019' '01/09/2019' '01/10/2019' '01/11/2019'
'01/12/2019' '01/14/2019' '01/15/2019' '01/16/2019' '01/17/2019'
'01/18/2019' '01/19/2019' '01/21/2019' '01/22/2019' '01/23/2019'
'01/24/2019' '01/25/2019' '01/26/2019' '01/28/2019' '01/29/2019'
'01/30/2019' '01/31/2019' '02/01/2019' '02/02/2019' '02/04/2019'
'02/05/2019' '02/06/2019' '02/07/2019' '02/08/2019' '02/09/2019'
'02/11/2019' '02/12/2019' '02/13/2019' '02/14/2019' '02/15/2019'
'02/16/2019' '02/19/2019' '02/20/2019' '02/21/2019' '02/22/2019'
'02/23/2019' '02/25/2019' '02/26/2019' '02/27/2019' '02/28/2019'
'03/01/2019' '03/02/2019' '03/03/2019' '03/04/2019' '03/05/2019'
'03/06/2019' '03/07/2019' '03/08/2019' '03/09/2019' '03/11/2019'
'03/12/2019' '03/13/2019' '03/14/2019' '03/15/2019' '03/16/2019'
'03/18/2019' '03/19/2019' '03/20/2019' '03/21/2019' '03/22/2019'
'03/23/2019' '03/25/2019' '03/26/2019' '03/27/2019' '03/28/2019'
'03/29/2019' '03/30/2019' '04/01/2019' '04/02/2019' '04/03/2019'
'04/04/2019' '04/05/2019' '04/06/2019' '04/08/2019' '04/09/2019'
'04/10/2019' '05/29/2018' '05/30/2018' '05/31/2018' '06/01/2018'
'06/02/2018' '06/04/2018' '06/05/2018' '06/06/2018' '06/07/2018'
'06/08/2018' '06/09/2018' '06/11/2018' '06/12/2018' '06/13/2018'
'06/14/2018' '06/15/2018' '06/16/2018' '06/18/2018' '06/19/2018'
'06/20/2018' '06/21/2018' '06/22/2018' '06/23/2018' '06/25/2018'
'06/26/2018' '06/27/2018' '06/28/2018' '06/29/2018' '06/30/2018'
'07/03/2018' '07/04/2018' '07/05/2018' '07/06/2018' '07/07/2018'
'07/09/2018' '07/10/2018' '07/11/2018' '07/12/2018' '07/13/2018'
'07/14/2018' '07/16/2018' '07/17/2018' '07/18/2018' '07/19/2018'
'07/20/2018' '07/21/2018' '07/23/2018' '07/24/2018' '07/25/2018'
'07/26/2018' '07/27/2018' '07/28/2018' '07/30/2018' '07/31/2018'
'08/01/2018' '08/02/2018' '08/03/2018' '08/04/2018' '08/07/2018'
'08/08/2018' '08/09/2018' '08/10/2018' '08/11/2018' '08/13/2018'
'08/14/2018' '08/15/2018' '08/16/2018' '08/17/2018' '08/18/2018'
'08/20/2018' '08/21/2018' '08/22/2018' '08/23/2018' '08/24/2018'
'08/25/2018' '08/27/2018' '08/28/2018' '08/29/2018' '08/30/2018'
'08/31/2018' '09/01/2018' '09/04/2018' '09/05/2018' '09/06/2018'
'09/07/2018' '09/08/2018' '09/10/2018' '09/11/2018' '09/12/2018'
'09/13/2018' '09/14/2018' '09/15/2018' '09/17/2018' '09/18/2018'
'09/19/2018' '09/20/2018' '09/21/2018' '09/22/2018' '09/24/2018'
'09/25/2018' '09/26/2018' '09/27/2018' '09/28/2018' '09/29/2018'
'10/01/2018' '10/02/2018' '10/03/2018' '10/04/2018' '10/05/2018'
'10/06/2018' '10/09/2018' '10/10/2018' '10/11/2018' '10/12/2018'
'10/13/2018' '10/15/2018' '10/16/2018' '10/17/2018' '10/18/2018'
'10/19/2018' '10/20/2018' '10/22/2018' '10/23/2018' '10/24/2018'
'10/25/2018' '10/26/2018' '10/29/2018' '10/30/2018' '10/31/2018'
'11/01/2018' '11/02/2018' '11/03/2018' '11/05/2018' '11/06/2018'
'11/07/2018' '11/08/2018' '11/09/2018' '11/10/2018' '11/13/2018'
'11/14/2018' '11/15/2018' '11/16/2018' '11/18/2018' '11/19/2018'
'11/20/2018' '11/21/2018' '11/22/2018' '11/23/2018' '11/24/2018'
'11/26/2018' '11/27/2018' '11/28/2018' '11/29/2018' '11/30/2018'
'12/01/2018' '12/03/2018' '12/04/2018' '12/05/2018' '12/06/2018'
'12/07/2018' '12/08/2018' '12/09/2018' '12/10/2018' '12/11/2018'
'12/12/2018' '12/13/2018' '12/14/2018' '12/15/2018' '12/17/2018'
'12/18/2018' '12/19/2018' '12/20/2018' '12/21/2018' '12/22/2018'
'12/24/2018' '12/25/2018' '12/27/2018' '12/28/2018' '12/29/2018'
'12/31/2018']
Somehow the data type of cursorDate is changed to Timestamp, making the value comparison not work.
How is it converting the datetime formats?
Building on my comment above. Change the last line before your loop to this: df_date = df['CDATE'].apply(pd.to_datetime).unique()