pandas.concat() does not fill the columns - python
I am trying to create dummy data as follows:
import numpy as np
import pandas as pd
def dummy_historical(seclist, dates, startvalues):
dfHist = pd.DataFrame(0, index=[0], columns=seclist)
for sec in seclist:
# (works fine)
svalue = startvalues[sec].max()
# this creates a random sequency of 84 rows and 1 column (works fine)
dfRandom = pd.DataFrame(np.random.randint(svalue-10,svalue+10, size=(dates.size, 1 )), index=dates, columns=[sec])
# does not work
dfHist[sec] = pd.concat([ dfHist[sec] , dfRandom ])
return dfHist
When I print dfHist, it only shows me the first row (as when initiated). Thus nothing has been filled.
Here is an example of the data:
seclist = ['AAPL', 'GOOGL']
# use any number for startvalues
dates = DatetimeIndex(['2017-01-05', '2017-01-06', '2017-01-07', '2017-01-08',
'2017-01-09', '2017-01-10', '2017-01-11', '2017-01-12',
'2017-01-13', '2017-01-14', '2017-01-15', '2017-01-16',
'2017-01-17', '2017-01-18', '2017-01-19', '2017-01-20',
'2017-01-21', '2017-01-22', '2017-01-23', '2017-01-24',
'2017-01-25', '2017-01-26', '2017-01-27', '2017-01-28',
'2017-01-29', '2017-01-30', '2017-01-31', '2017-02-01',
'2017-02-02', '2017-02-03', '2017-02-04', '2017-02-05',
'2017-02-06', '2017-02-07', '2017-02-08', '2017-02-09',
'2017-02-10', '2017-02-11', '2017-02-12', '2017-02-13',
'2017-02-14', '2017-02-15', '2017-02-16', '2017-02-17',
'2017-02-18', '2017-02-19', '2017-02-20', '2017-02-21',
'2017-02-22', '2017-02-23', '2017-02-24', '2017-02-25',
'2017-02-26', '2017-02-27', '2017-02-28', '2017-03-01',
'2017-03-02', '2017-03-03', '2017-03-04', '2017-03-05',
'2017-03-06', '2017-03-07', '2017-03-08', '2017-03-09',
'2017-03-10', '2017-03-11', '2017-03-12', '2017-03-13',
'2017-03-14', '2017-03-15', '2017-03-16', '2017-03-17',
'2017-03-18', '2017-03-19', '2017-03-20', '2017-03-21',
'2017-03-22', '2017-03-23', '2017-03-24', '2017-03-25',
'2017-03-26', '2017-03-27', '2017-03-28', '2017-03-29'],
dtype='datetime64[ns]', freq='D')
You need to pass axis=1 to concat if you want to concatenate columns. In addition, you don't need to initialize your data frame with data in the beginning (except you want to have the 0 value):
def dummy_historical(seclist, dates, startvalues):
dfHist = pd.DataFrame()
for sec in seclist:
svalue = startvalues[sec].max()
dfRandom = pd.DataFrame(np.random.randint(svalue-10,svalue+10, size=(dates.size, 1 )), index=dates, columns=[sec])
dfHist = pd.concat([ dfHist , dfRandom ], axis=1)
return dfHist
You can even write in a more concise way avoiding concat like:
def generate(sec):
svalue = startvalues[sec].max()
return np.random.randint(svalue-10,svalue+10, size=dates.size)
dfHist = pd.DataFrame({sec: generate(sec) for sec in seclist}, index=dates)
Related
Split a CSV in three parts and calculating the mean
I have a file containing:
Time 60Ni 61Ni 62Ni 63Cu 64Ni 65Cu 66Zn
0. 9.13242244720459 0.406570166349411 1.326429009437561 5.754200458526611 0.4233334958553314 2.68562912940979 4.148788005113602e-002
8.390999794006348 9.187464714050293 0.4089393615722656 1.334462523460388 5.790649890899658 0.425884485244751 2.702604055404663 4.17313240468502e-002
16.78300094604492 9.254316329956055 0.4119723737239838 1.344084143638611 5.832504749298096 0.428943395614624 2.722275018692017 4.203101620078087e-002
25.17399978637695 9.19857120513916 0.4094997346401215 1.336091756820679 5.791898727416992 0.4264563024044037 2.703336715698242 4.185733571648598e-002
33.56499862670898 9.194388389587402 0.4092871248722076 1.335391044616699 5.794968605041504 0.4264419078826904 2.704529047012329 4.192239791154862e-002
41.95600128173828 9.162041664123535 0.4078944325447083 1.330722570419312 5.766440868377686 0.425002932548523 2.691519498825073 4.182799160480499e-002
50.34700012207031 9.190646171569824 0.4091125726699829 1.334963202476502 5.786285877227783 0.426413893699646 2.700882434844971 4.196327552199364e-002
58.73799896240234 9.211565971374512 0.4100649058818817 1.337916374206543 5.8003830909729 0.4273969829082489 2.707314252853394 4.207673668861389e-002
67.12799835205078 9.240947723388672 0.4113766849040985 1.342136979103088 5.822870254516602 0.4287911653518677 2.717630624771118 4.222121462225914e-002
75.51899719238281 9.208130836486816 0.4099342525005341 1.337505698204041 5.802256584167481 0.4273860156536102 2.708084583282471 4.214133694767952e-002
83.91000366210938 9.196262359619141 0.4093911945819855 1.335786700248718 5.799176692962647 0.4268693923950195 2.706451416015625 4.215647280216217e-002
92.30100250244141 9.213265419006348 0.4101545214653015 1.338128447532654 5.807514190673828 0.4277283549308777 2.71068549156189 4.221603646874428e-002
100.6920013427734 9.163029670715332 0.407885879278183 1.330831050872803 5.775251865386963 0.4254410266876221 2.695534229278565 4.204751178622246e-002
109.0839996337891 9.144490242004395 0.4070722758769989 1.328153848648071 5.764679908752441 0.4246650040149689 2.690402746200562 4.198652133345604e-002
117.4749984741211 9.114171028137207 0.4057718515396118 1.32369875907898 5.745044231414795 0.4233448505401611 2.681406497955322 4.190905019640923e-002
125.8659973144531 9.149589538574219 0.407274603843689 1.328810453414917 5.766050815582275 0.4248199760913849 2.691139459609985 4.200970754027367e-002
134.2570037841797 9.168668746948242 0.4081465899944305 1.331702351570129 5.777794361114502 0.4256783723831177 2.696741819381714 4.206346347928047e-002
142.6479949951172 9.11380672454834 0.4057287871837616 1.323864817619324 5.740524291992188 0.4232001006603241 2.67945122718811 4.187140986323357e-002
151.0390014648438 9.100893974304199 0.4051263332366943 1.321851253509522 5.729655265808106 0.4226666390895844 2.674278259277344 4.182597994804382e-002
159.4299926757813 9.072731971740723 0.4039073586463928 1.317763328552246 5.713830471038818 0.4213792979717255 2.666974782943726 4.169051349163055e-002
167.8209991455078 9.186164855957031 0.4089057147502899 1.334116697311401 5.786634922027588 0.4264728426933289 2.700879812240601 4.211126267910004e-002
176.2129974365234 9.13982105255127 0.4068569839000702 1.327479124069214 5.76115083694458 0.4244593381881714 2.688895463943481 4.199059307575226e-002
184.60400390625 9.146007537841797 0.4071221053600311 1.328468441963196 5.762693881988525 0.4247534275054932 2.689634084701538 4.1985172778368e-002
192.9949951171875 9.18150806427002 0.4086942672729492 1.333438873291016 5.785679817199707 0.4262394905090332 2.700178623199463 4.207265004515648e-002
201.3860015869141 9.134004592895508 0.4066038727760315 1.326677560806274 5.753909587860107 0.424109697341919 2.685543775558472 4.191514849662781e-002
209.7769927978516 9.192599296569824 0.4091922044754028 1.335113883018494 5.792657852172852 0.4266164898872376 2.703598737716675 4.208896681666374e-002
218.1679992675781 9.166966438293457 0.4080702364444733 1.331447958946228 5.776984214782715 0.4254603683948517 2.696239709854126 4.19912114739418e-002
226.5590057373047 9.166423797607422 0.4080766439437866 1.331416010856628 5.771696090698242 0.4254250526428223 2.693812847137451 4.191195592284203e-002
234.9510040283203 9.122139930725098 0.4060815274715424 1.325031995773315 5.74381160736084 0.4234589040279388 2.680959224700928 4.174426198005676e-002
243.3419952392578 9.178729057312012 0.4085982143878937 1.333097338676453 5.783432006835938 0.4259471595287323 2.699411153793335 4.196531698107719e-002
251.7330017089844 9.196023941040039 0.4093179702758789 1.335668444633484 5.792133331298828 0.4266210496425629 2.703416347503662 4.196692258119583e-002
260.1239929199219 9.195613861083984 0.4093446731567383 1.33561098575592 5.790852546691895 0.4264806509017944 2.702755451202393 4.19374406337738e-002
268.5150146484375 9.124658584594727 0.4061901867389679 1.325218439102173 5.749895572662354 0.4233379364013672 2.683579206466675 4.166891798377037e-002
276.906005859375 9.071592330932617 0.4038631021976471 1.317633748054504 5.711780071258545 0.4209088683128357 2.666091680526733 4.146279022097588e-002
285.2969970703125 9.090703010559082 0.4047099351882935 1.320350289344788 5.724553108215332 0.4218063056468964 2.671880960464478 4.148663952946663e-002
293.68798828125 9.049410820007324 0.4028385281562805 1.314435601234436 5.699662208557129 0.4198987782001495 2.660340070724487 4.135752841830254e-002
302.0790100097656 9.158493995666504 0.4077092707157135 1.330130934715271 5.770212650299072 0.4247544705867767 2.693133354187012 4.172087088227272e-002
310.4700012207031 9.294267654418945 0.4137440025806427 1.350019454956055 5.85582971572876 0.4307662844657898 2.733232498168945 4.217509180307388e-002
318.8609924316406 9.266000747680664 0.4124558866024017 1.34581983089447 5.838682651519775 0.429353654384613 2.724989175796509 4.206011816859245e-002
327.2520141601563 9.227903366088867 0.4107420146465302 1.340180039405823 5.813295841217041 0.4277106523513794 2.713207006454468 4.191378504037857e-002
335.6430053710938 9.248990058898926 0.4117128551006317 1.343235015869141 5.836093425750732 0.4286618232727051 2.72357988357544 4.200825467705727e-002
344.0339965820313 9.200018882751465 0.4095089137554169 1.336208343505859 5.805673122406006 0.4264824092388153 2.709526300430298 4.185647144913673e-002
352.4259948730469 9.162602424621582 0.4079090356826782 1.330750703811646 5.780079364776611 0.4248281121253967 2.697546243667603 4.17003221809864e-002
360.8169860839844 9.165441513061523 0.4079831540584564 1.331099987030029 5.780121326446533 0.424967348575592 2.697607517242432 4.169800505042076e-002
369.2070007324219 9.242767333984375 0.4114582240581513 1.342459917068481 5.828019142150879 0.4283893704414368 2.719994068145752 4.194791615009308e-002
377.5989990234375 9.211434364318848 0.4100139439105988 1.337894320487976 5.801908493041992 0.4268820583820343 2.708046913146973 4.185103997588158e-002
385.989990234375 9.168110847473145 0.4081266224384308 1.33171010017395 5.772421360015869 0.4250668585300446 2.694308280944824 4.166359454393387e-002
394.3810119628906 9.162002563476563 0.4078731238842011 1.330778479576111 5.770648956298828 0.4247135519981384 2.693532466888428 4.165602847933769e-002
402.7720031738281 9.219051361083984 0.4104039072990418 1.339054584503174 5.805272579193115 0.4273586571216583 2.709418296813965 4.186749085783958e-002
411.1640014648438 9.225748062133789 0.4106448590755463 1.340008854866028 5.808595180511475 0.4276045560836792 2.711185216903687 4.189140349626541e-002
425.0020141601563 9.11283016204834 0.4056265950202942 1.323553919792175 5.742629528045654 0.4226277768611908 2.680011749267578 4.150775447487831e-002
433.3930053710938 9.15496826171875 0.4075464010238648 1.329663395881653 5.76693058013916 0.4244976043701172 2.691663980484009 4.165017232298851e-002
441.7839965820313 9.179342269897461 0.4086317718029022 1.333258748054504 5.783347606658936 0.4256252646446228 2.699387073516846 4.177364706993103e-002
450.1759948730469 9.202337265014648 0.4096647799015045 1.336641907691956 5.799064636230469 0.4267286956310272 2.706497669219971 4.189135506749153e-002
458.5669860839844 9.126877784729004 0.4062632024288178 1.325594425201416 5.7450852394104 0.4234336316585541 2.681554317474365 4.164514690637589e-002
466.9580078125 9.130221366882324 0.4063588082790375 1.326080322265625 5.750959873199463 0.4235436022281647 2.6843581199646 4.169851914048195e-002
475.3489990234375 9.142138481140137 0.4069503247737885 1.32788360118866 5.753814697265625 0.4240946471691132 2.685687065124512 4.17218841612339e-002
483.739990234375 9.144487380981445 0.4070816040039063 1.328163623809815 5.764283180236816 0.4243338704109192 2.69016432762146 4.180238768458366e-002
492.1310119628906 9.213832855224609 0.4101627767086029 1.338177442550659 5.806262969970703 0.4273685812950134 2.709989309310913 4.204079136252403e-002
500.5220031738281 9.151962280273438 0.4073929488658905 1.329235196113586 5.765473365783691 0.4247141480445862 2.691080808639526 4.187702387571335e-002
508.9129943847656 9.133262634277344 0.4065472185611725 1.326548576354981 5.755089282989502 0.4239353835582733 2.685916900634766 4.184074699878693e-002
517.3040161132813 9.194231033325195 0.4092318415641785 1.335361480712891 5.791540622711182 0.4266365468502045 2.703181505203247 4.204431921243668e-002
525.6950073242188 9.174141883850098 0.4084053635597229 1.332433700561523 5.780707836151123 0.4258663356304169 2.697983264923096 4.203671962022781e-002
534.0869750976563 9.127938270568848 0.4063973724842072 1.325674772262573 5.753820896148682 0.4238673448562622 2.685414791107178 4.189241677522659e-002
542.4769897460938 9.228574752807617 0.4108735322952271 1.340509295463562 5.816771030426025 0.4283493161201477 2.714869976043701 4.227539896965027e-002
550.8679809570313 9.247261047363281 0.4116438031196594 1.34306275844574 5.829936504364014 0.4292499721050263 2.720824480056763 4.234698414802551e-002
559.2589721679688 9.259587287902832 0.4121484756469727 1.344773530960083 5.840207099914551 0.4296930134296417 2.725474834442139 4.239725694060326e-002
567.6500244140625 9.236879348754883 0.4112152457237244 1.341552734375 5.824738502502441 0.4288162887096405 2.718418121337891 4.232741147279739e-002
576.041015625 9.265199661254883 0.4123806655406952 1.345624566078186 5.837865352630615 0.4300332069396973 2.724727630615234 4.243086278438568e-002
584.4310302734375 9.193467140197754 0.4092609882354736 1.335316061973572 5.791056632995606 0.4267773926258087 2.702801465988159 4.214197397232056e-002
592.822021484375 9.178906440734863 0.408621221780777 1.333141565322876 5.783803462982178 0.4262367188930512 2.699366569519043 4.21367958188057e-002
601.2139892578125 9.179999351501465 0.4086976051330566 1.333412766456604 5.781562805175781 0.4262183606624603 2.698424100875855 4.212524741888046e-002
609.60498046875 9.158502578735352 0.4077076315879822 1.330240249633789 5.771774768829346 0.4252981841564179 2.693920612335205 4.206201061606407e-002
617.9949951171875 9.168906211853027 0.4081432521343231 1.331776857376099 5.777164459228516 0.4257596433162689 2.696363210678101 4.212769865989685e-002
626.385986328125 9.148199081420898 0.4072228968143463 1.328739166259766 5.764687061309815 0.4248482882976532 2.690601110458374 4.204926639795303e-002
634.7769775390625 9.153997421264648 0.4075290560722351 1.329600691795349 5.76605749130249 0.4250805974006653 2.691195011138916 4.203818738460541e-002
643.1680297851563 9.142102241516113 0.4070025384426117 1.327812790870667 5.758194923400879 0.4244733154773712 2.687539577484131 4.197685047984123e-002
651.5599975585938 9.157526016235352 0.4076575040817261 1.33014190196991 5.771289825439453 0.4252424538135529 2.693483829498291 4.207025840878487e-002
659.9509887695313 9.142055511474609 0.4069408476352692 1.327834606170654 5.75890064239502 0.4245132505893707 2.687950849533081 4.196911677718163e-002
668.3410034179688 9.163941383361816 0.4079061448574066 1.331052899360657 5.773416519165039 0.425525963306427 2.694749593734741 4.208214208483696e-002
676.7329711914063 9.214210510253906 0.4101268947124481 1.338269472122192 5.804011821746826 0.4277287721633911 2.70874834060669 4.224084317684174e-002
685.1240234375 9.221725463867188 0.410546600818634 1.33942449092865 5.808478832244873 0.4280569553375244 2.710729837417603 4.224072396755219e-002
693.5139770507813 9.195225715637207 0.4093619287014008 1.335615515708923 5.792295932769775 0.4269255101680756 2.703481912612915 4.215554893016815e-002
701.905029296875 9.236662864685059 0.4111031889915466 1.341474533081055 5.820279121398926 0.4286713898181915 2.716408491134644 4.231745004653931e-002
710.2969970703125 9.219303131103516 0.4103749394416809 1.33903431892395 5.809108257293701 0.4279004633426666 2.711240530014038 4.220414161682129e-002
718.68798828125 9.196757316589356 0.4093507528305054 1.335767865180969 5.794125556945801 0.4269102811813355 2.704240798950195 4.217429086565971e-002
727.0789794921875 9.169294357299805 0.4081831276416779 1.331677913665772 5.778267860412598 0.4257012009620667 2.696781396865845 4.20493595302105e-002
735.468994140625 9.254044532775879 0.4119507372379303 1.344122529029846 5.83418083190918 0.4294586181640625 2.722884654998779 4.238997399806976e-002
743.8610229492188 9.224509239196777 0.4105926156044006 1.339867234230042 5.812450408935547 0.4280983507633209 2.712637424468994 4.227783530950546e-002
752.2520141601563 9.167038917541504 0.4080414175987244 1.331365466117859 5.778883457183838 0.4256396591663361 2.697120428085327 4.206839948892593e-002
760.6430053710938 9.156136512756348 0.407585471868515 1.329828977584839 5.771244049072266 0.4251766502857208 2.693709135055542 4.204395413398743e-002
769.0339965820313 9.206752777099609 0.4098866879940033 1.337259769439697 5.798995018005371 0.4273804128170013 2.706660270690918 4.218916967511177e-002
777.4249877929688 9.185664176940918 0.4088890254497528 1.33407187461853 5.787529468536377 0.426471084356308 2.701387643814087 4.21074777841568e-002
785.8159790039063 9.148477554321289 0.4072705209255219 1.328797459602356 5.764423847198486 0.4247606992721558 2.690322160720825 4.200183600187302e-002
794.2069702148438 9.139849662780762 0.4068310558795929 1.327486157417297 5.760977268218994 0.4244396984577179 2.688838005065918 4.198827594518662e-002
802.5980224609375 9.198716163635254 0.409488320350647 1.336077690124512 5.797767639160156 0.4270517528057098 2.705855131149292 4.215721413493156e-002
810.989013671875 9.175697326660156 0.4084174335002899 1.332631826400757 5.781099796295166 0.425992488861084 2.698201894760132 4.206936806440353e-002
819.3800048828125 9.106189727783203 0.4053537547588348 1.322664737701416 5.740387916564941 0.4229016602039337 2.679165840148926 4.18708510696888e-002
827.77099609375 9.11962890625 0.4059470593929291 1.324671149253845 5.745753765106201 0.4235488474369049 2.681836843490601 4.189123585820198e-002
836.1619873046875 9.221225738525391 0.4104022979736328 1.33923864364624 5.813970565795898 0.4279847741127014 2.713436365127564 4.224034398794174e-002
849.9970092773438 9.109155654907227 0.4055195748806 1.323018074035645 5.738785743713379 0.4229097962379456 2.678738832473755 4.17560487985611e-002
858.3880004882813 9.081585884094238 0.4043126106262207 1.319140315055847 5.720804691314697 0.4216950535774231 2.670202732086182 4.168836399912834e-002
866.7789916992188 9.1737060546875 0.4083895683288574 1.332486510276794 5.779799461364746 0.4258598983287811 2.697497129440308 4.201843962073326e-002
875.1699829101563 9.215715408325195 0.4102407991886139 1.33849024772644 5.806502342224121 0.4276199042797089 2.710031509399414 4.214433580636978e-002
883.5609741210938 9.29750919342041 0.4138506650924683 1.350215315818787 5.858696460723877 0.4313125610351563 2.734477758407593 4.240995645523071e-002
891.9520263671875 9.251111030578613 0.411830872297287 1.343641996383667 5.826048374176025 0.4292575418949127 2.719125270843506 4.226363822817802e-002
900.343017578125 9.236968994140625 0.411191999912262 1.341637492179871 5.816394329071045 0.4285323023796082 2.71470046043396 4.218020662665367e-002
908.7340087890625 9.18012809753418 0.4086549580097199 1.333361864089966 5.780932903289795 0.4260410964488983 2.698340177536011 4.198113456368446e-002
917.125 9.18910026550293 0.4090204238891602 1.334587931632996 5.791236877441406 0.426427572965622 2.702847242355347 4.205641150474548e-002
925.5159912109375 9.163248062133789 0.4078385829925537 1.330891489982605 5.775006771087647 0.4252764880657196 2.695378065109253 4.195348545908928e-002
933.906982421875 9.184928894042969 0.4089162349700928 1.334069848060608 5.789799213409424 0.42618727684021 2.702196598052979 4.199947416782379e-002
942.2979736328125 9.157343864440918 0.4076671004295349 1.330055475234985 5.770273208618164 0.4249707460403442 2.693178653717041 4.188660532236099e-002
950.6890258789063 9.162631988525391 0.4078827202320099 1.330793499946594 5.77417516708374 0.4251722097396851 2.695005416870117 4.190302640199661e-002
959.0800170898438 9.114273071289063 0.4057436585426331 1.323749780654907 5.743786811828613 0.4230408370494843 2.680756568908691 4.173881560564041e-002
967.4710083007813 9.244811058044434 0.4115355014801025 1.34266197681427 5.823981761932373 0.4288525879383087 2.718071460723877 4.214448481798172e-002
975.8619995117188 9.219685554504395 0.4104566872119904 1.339130640029907 5.808487892150879 0.4276332259178162 2.710957288742065 4.206658154726028e-002
984.2529907226563 9.184207916259766 0.4088565707206726 1.33392071723938 5.792478561401367 0.4260831475257874 2.703508853912354 4.195259138941765e-002
992.6439819335938 9.13871955871582 0.4068254828453064 1.327333569526672 5.761001586914063 0.4240987598896027 2.688708066940308 4.179005324840546e-002
1001.034973144531 9.151439666748047 0.4073895514011383 1.329284429550171 5.767615795135498 0.4246693849563599 2.691930532455444 4.182363301515579e-002
1009.424987792969 9.19940185546875 0.409492164850235 1.335996866226196 5.800271034240723 0.4267957508563995 2.70706057548523 4.198677837848663e-002
1017.815979003906 9.255974769592285 0.4120437800884247 1.344139099121094 5.840244770050049 0.4293366670608521 2.725528001785278 4.220050573348999e-002
1026.20703125 9.220073699951172 0.4104630351066589 1.339051723480225 5.81441593170166 0.4276903867721558 2.713610172271729 4.208677262067795e-002
1034.598022460938 9.158895492553711 0.4077011644840241 1.330096125602722 5.776969432830811 0.4249850511550903 2.696006536483765 4.186514392495155e-002
1042.989013671875 9.135567665100098 0.4066715240478516 1.326890826225281 5.756415843963623 0.423865556716919 2.686625719070435 4.174899682402611e-002
1051.380981445313 9.150594711303711 0.4073532521724701 1.329049825668335 5.765689849853516 0.4245824813842773 2.691075325012207 4.179978370666504e-002
1059.77197265625 9.146571159362793 0.4071609079837799 1.32847785949707 5.760791778564453 0.4242803156375885 2.688825607299805 4.17768582701683e-002
1068.162963867188 9.131063461303711 0.4064978063106537 1.326229453086853 5.752644538879395 0.4236991405487061 2.684972286224365 4.172741994261742e-002
1076.553955078125 9.098221778869629 0.4049918949604034 1.321496725082398 5.731342792510986 0.4222320318222046 2.675036668777466 4.162869602441788e-002
1084.944946289063 9.169441223144531 0.4081719219684601 1.331780910491943 5.776838779449463 0.4254011511802673 2.696260452270508 4.184866324067116e-002
1093.337036132813 9.187003135681152 0.4089777171611786 1.334323048591614 5.790809154510498 0.4261792898178101 2.702747344970703 4.196572676301003e-002
1101.72802734375 9.179986953735352 0.4086208045482636 1.333386778831482 5.783829689025879 0.4258585274219513 2.699674844741821 4.191147163510323e-002
1110.119018554688 9.200528144836426 0.4095506370067596 1.336296439170837 5.797418117523193 0.4267379641532898 2.7057945728302 4.19546514749527e-002
1118.509033203125 9.158334732055664 0.4076752066612244 1.330214262008667 5.770383834838867 0.4248470067977905 2.693165063858032 4.180992022156715e-002
1126.900024414063 9.194581985473633 0.4093466997146606 1.335410833358765 5.798298358917236 0.4264914393424988 2.706053495407105 4.194727912545204e-002
1135.291015625 9.176510810852051 0.4084961414337158 1.3328697681427 5.778421401977539 0.4256733357906342 2.697108507156372 4.18514646589756e-002
1143.682983398438 9.163573265075684 0.4079014360904694 1.330968260765076 5.773004055023193 0.4250616133213043 2.694518804550171 4.183558747172356e-002
1152.072998046875 9.159396171569824 0.4077317416667938 1.330322265625 5.771379947662354 0.4248954653739929 2.693806171417236 4.181275144219399e-002
1160.464965820313 9.165866851806641 0.4080128371715546 1.331347465515137 5.772171497344971 0.4252021610736847 2.694234848022461 4.181317612528801e-002
1168.85595703125 9.151269912719727 0.407374233007431 1.329119086265564 5.760807991027832 0.424500435590744 2.688781023025513 4.176882281899452e-002
1177.246948242188 9.141792297363281 0.4069608747959137 1.327713966369629 5.75624418258667 0.4241056740283966 2.68661379814148 4.173726961016655e-002
1185.636962890625 9.130838394165039 0.406494677066803 1.326230525970459 5.751668930053711 0.4236221015453339 2.684362649917603 4.168353974819183e-002
1194.027954101563 9.206241607666016 0.4098086059093475 1.337079763412476 5.802299022674561 0.4269396662712097 2.707928895950317 4.194400832056999e-002
1202.4189453125 9.17149543762207 0.4083086550235748 1.332085609436035 5.776546001434326 0.4253532886505127 2.696049451828003 4.180750250816345e-002
1210.81005859375 9.140050888061523 0.4068616330623627 1.327504873275757 5.760209083557129 0.4239790141582489 2.6883225440979 4.170787334442139e-002
1219.201049804688 9.165439605712891 0.4079880714416504 1.331203103065491 5.77871561050415 0.4250532984733582 2.697003841400147 4.180311039090157e-002
1227.593017578125 9.177500724792481 0.4085498750209808 1.332932233810425 5.783236026763916 0.4255987405776978 2.699163913726807 4.181493073701859e-002
1235.984008789063 9.177756309509277 0.408606618642807 1.33305811882019 5.782862663269043 0.4256067276000977 2.699074268341065 4.182154312729836e-002
1244.375 9.143049240112305 0.4070280194282532 1.327925682067871 5.766200542449951 0.4240804016590118 2.691066265106201 4.171686246991158e-002
1252.765991210938 9.110544204711914 0.4055243730545044 1.323151469230652 5.742761135101318 0.422651082277298 2.680213212966919 4.159015789628029e-002
1261.156982421875 9.153350830078125 0.4074757993221283 1.329340934753418 5.772144794464111 0.4244934320449829 2.693885564804077 4.173129424452782e-002
I want to split the file in three parts รก 50 rows:
data = pd.read_csv(file, sep='\t', names=['Time', '60Ni', '61Ni', '62Ni', '63Cu', '64Ni', '65Cu', '66Zn'], skiprows=3, nrows=50, index_col=False, dtype=float)
data2 = pd.read_csv(file, sep='\t', names=['Time', '60Ni', '61Ni', '62Ni', '63Cu', '64Ni', '65Cu', '66Zn'], skiprows=53, nrows=50, index_col=False, dtype=float)
data3 = pd.read_csv(file, sep='\t', names=['Time', '60Ni', '61Ni', '62Ni', '63Cu', '64Ni', '65Cu', '66Zn'], skiprows=103, nrows=50, index_col=False, dtype=float)
Then I'm removing outliers with:
cols = list(data.drop(columns='Time').columns)
datao = pd.DataFrame({'Time':data['Time']})
datao[cols] = data[cols].where(np.abs(stats.zscore(data[cols])) < 2)
cols = list(data2.drop(columns='Time').columns)
data2o = pd.DataFrame({'Time':data2['Time']})
data2o[cols] = data2[cols].where(np.abs(stats.zscore(data2[cols])) < 2)
data2o[cols] = data2o[cols].mean()
cols = list(data3.drop(columns='Time').columns)
data3o = pd.DataFrame({'Time':data3['Time']})
data3o[cols] = data3[cols].where(np.abs(stats.zscore(data3[cols])) < 2)
data3o[cols] = data3o[cols].mean()
Does this make sense so far?
And now I would like to create a mean of datao, data2o and data3o seperately, resulting in three values for 60Ni, 61Ni, 62Ni, 63Cu, 64Ni, 65Cu, 66Zn. After that, I want to make a mean of these three values again. How should I do this?
I tried to make it this way:
mean_filtered_transposed = pd.DataFrame(data=np.mean(data)).T
mean_filtered_transposed['Time'] = pd.to_datetime(mean_filtered_transposed["Time"], unit='s')mean_filtered_transposed = pd.DataFrame(data=np.mean(data)).T
mean_filtered_transposed['Time'] = pd.to_datetime(mean_filtered_transposed["Time"], unit='s')
mean_filtered_transposed2 = pd.DataFrame(data=np.mean(data2)).T
mean_filtered_transposed2['Time'] = pd.to_datetime(mean_filtered_transposed["Time"], unit='s')
mean_filtered_transposed3 = pd.DataFrame(data=np.mean(data3)).T
mean_filtered_transposed3['Time'] = pd.to_datetime(mean_filtered_transposed3["Time"], unit='s')
mean_all = pd.concat(mean_filtered_transposed, mean_filtered_transposed2, mean_filtered_transposed3)
However, this results in:
"TypeError: first argument must be an iterable of pandas objects, you passed an object of type "DataFrame""
Based on documentation: objs: a sequence or mapping of Series or DataFrame objects So: s1 = pd.Series(['a', 'b']) s2 = pd.Series(['c', 'd']) pd.concat([s1, s2]) result: But: s1 = pd.Series(['a', 'b']) s2 = pd.Series(['c', 'd']) pd.concat(s1, s2) generates:
Correlating values of dictionary - 'dict' object has no attribute 'corr'
I am new to python, I am trying to work with dictionary and correlation. I am trying to get a better understanding of working with dictionaries. But I get an error when I try to correlate the dictionary values for every year, so 2010 up to 2021.
data = {}
for year in names:
tableName = category+str(year)
n = ', '.join(f'"{name}"' for name in names[year])
sqlQ = f'select "Date", {n} from {tableName};'
cur.execute(sqlQ)
res = cur.fetchall()
data[year] = {}
for row in res:
if row[0] in dateRange and row[0].year == year:
date = row[0].strftime("%d/%m/%Y")
data[year][date] = []
for i in range(1, len(row)):
data[year][date].append(float(row[i]))
data
{2010: {'01/01/2010': [18.16,
59.73,
218.41,
101.14,
44.15,
10.0,
134.52,
40.15,
7.53,
18.19,
24.2,
10.38,
25.6,
48.17,
18.44,
32.1,
9.11,
39.73,
88.16,
23.97,
14.19,
15.34,
45.86,
24.25,
39.83,
43.46,
8.07,
9.8,
7.87,
32.73,
13.57,
39.04,
18.68,
11.55,
22.67,
52.97,
106.15,
47.49,
34.16,
26.67,
61.96,
54.09,
7.24,
14.94,
78.03,
19.31,
34.36,
10.9,
10.73,
35.27,
16.1,
37.61,
16.61,
48.65,
38.12,
26.78,
104.99,
26.73,
22.46,
30.68,
30.7,
65.09,
50.28,
80.31,
19.91,
32.27,
62.5,
6.03,
9.14,
16.16,
24.53,
33.29,
53.25,
168.84,
10.68,
130.9,
30.12,
23.01,
23.49,
80.66,
58.73,
9.15,
16.52,
96.83,
35.17,
15.11,
21.64,
23.39,
24.24,
26.93,
42.85,
24.18,
61.75,
80.98,
30.48,
40.68,
26.06,
3.81,
30.5,
13.9], '04/01/2010': [18.37,
61.71,
223.96,
102.92,
45.26,
10.28,
133.9,
41.73,
7.64,
18.89,
24.28,
10.38,
25.68, ...
import pandas as pd
df = data
for key, value in df.items():
#print(key,value)
df.corr(method ='pearson')
AttributeError: 'dict' object has no attribute 'corr'
You had to store the dictionary in a pandas's dataframe df = pd.DataFrame(data) import pandas as pd for key, value in data.items(): print(key) df = pd.DataFrame(value) print(df.corr(method ='pearson'))
Select specific dates from a data frame in python using MODIS data in NETCDF4
I am not well-versed in python, and I'm sure there is a simple solution to this (although, I have looked). I got this code from an lpdaac tutorial.
My input is a NETCDF4 file downloaded from MODIS satellite. Printing the metadata of the file returns the variables
file_in = Dataset(file_list[0], 'r', format = 'NETCDF4')
#print metadata
list(file_in.variables)
Out[19]: ['crs', 'time', 'lat', 'lon', '_1_km_16_days_EVI', '_1_km_16_days_VI_Quality']
I want to convert the time variable to date format, and then only select 1 date from each year. Here is the code to convert to date format:
from netCDF4 import num2date
times = file_in.variables["time"] #import time variables
dates = num2date(times[:], times.units) #get the time info
dates = [date.strftime("%Y-%m-%d") for date in dates] #get the list of datetime
print(dates)
The dates are as follows:
['2000-06-25', '2000-07-11', '2000-07-27', '2000-08-12', '2000-08-28', '2000-09-13', '2000-09-29', '2000-10-15', '2000-10-31', '2000-11-16', '2000-12-02', '2000-12-18', '2001-01-01', '2001-01-17', '2001-02-02', '2001-02-18', '2001-03-06', '2001-03-22', '2001-04-07', '2001-04-23', '2001-05-09', '2001-05-25', '2001-06-10', '2001-06-26', '2001-07-12', '2001-07-28', '2001-08-13', '2001-08-29', '2001-09-14', '2001-09-30', '2001-10-16', '2001-11-01', '2001-11-17', '2001-12-03', '2001-12-19', '2002-01-01', '2002-01-17', '2002-02-02', '2002-02-18', '2002-03-06', '2002-03-22', '2002-04-07', '2002-04-23', '2002-05-09', '2002-05-25', '2002-06-10', '2002-06-26', '2002-07-12', '2002-07-28', '2002-08-13', '2002-08-29', '2002-09-14', '2002-09-30', '2002-10-16', '2002-11-01', '2002-11-17', '2002-12-03', '2002-12-19', '2003-01-01', '2003-01-17', '2003-02-02', '2003-02-18', '2003-03-06', '2003-03-22', '2003-04-07', '2003-04-23', '2003-05-09', '2003-05-25', '2003-06-10', '2003-06-26', '2003-07-12', '2003-07-28', '2003-08-13', '2003-08-29', '2003-09-14', '2003-09-30', '2003-10-16', '2003-11-01', '2003-11-17', '2003-12-03', '2003-12-19', '2004-01-01', '2004-01-17', '2004-02-02', '2004-02-18', '2004-03-05', '2004-03-21', '2004-04-06', '2004-04-22', '2004-05-08', '2004-05-24', '2004-06-09', '2004-06-25', '2004-07-11', '2004-07-27', '2004-08-12', '2004-08-28', '2004-09-13', '2004-09-29', '2004-10-15', '2004-10-31', '2004-11-16', '2004-12-02', '2004-12-18', '2005-01-01', '2005-01-17', '2005-02-02', '2005-02-18', '2005-03-06', '2005-03-22', '2005-04-07', '2005-04-23', '2005-05-09', '2005-05-25', '2005-06-10', '2005-06-26', '2005-07-12', '2005-07-28', '2005-08-13', '2005-08-29', '2005-09-14', '2005-09-30', '2005-10-16', '2005-11-01', '2005-11-17', '2005-12-03', '2005-12-19', '2006-01-01', '2006-01-17', '2006-02-02', '2006-02-18', '2006-03-06', '2006-03-22', '2006-04-07', '2006-04-23', '2006-05-09', '2006-05-25', '2006-06-10', '2006-06-26', '2006-07-12', '2006-07-28', '2006-08-13', '2006-08-29', '2006-09-14', '2006-09-30', '2006-10-16', '2006-11-01', '2006-11-17', '2006-12-03', '2006-12-19', '2007-01-01', '2007-01-17', '2007-02-02', '2007-02-18', '2007-03-06', '2007-03-22', '2007-04-07', '2007-04-23', '2007-05-09', '2007-05-25', '2007-06-10', '2007-06-26', '2007-07-12', '2007-07-28', '2007-08-13', '2007-08-29', '2007-09-14', '2007-09-30', '2007-10-16', '2007-11-01', '2007-11-17', '2007-12-03', '2007-12-19', '2008-01-01', '2008-01-17', '2008-02-02', '2008-02-18', '2008-03-05', '2008-03-21', '2008-04-06', '2008-04-22', '2008-05-08', '2008-05-24', '2008-06-09', '2008-06-25', '2008-07-11', '2008-07-27', '2008-08-12', '2008-08-28', '2008-09-13', '2008-09-29', '2008-10-15', '2008-10-31', '2008-11-16', '2008-12-02', '2008-12-18', '2009-01-01', '2009-01-17', '2009-02-02', '2009-02-18', '2009-03-06', '2009-03-22', '2009-04-07', '2009-04-23', '2009-05-09', '2009-05-25', '2009-06-10', '2009-06-26', '2009-07-12', '2009-07-28', '2009-08-13', '2009-08-29', '2009-09-14', '2009-09-30', '2009-10-16', '2009-11-01', '2009-11-17', '2009-12-03', '2009-12-19', '2010-01-01', '2010-01-17', '2010-02-02', '2010-02-18', '2010-03-06', '2010-03-22', '2010-04-07', '2010-04-23', '2010-05-09', '2010-05-25', '2010-06-10', '2010-06-26', '2010-07-12', '2010-07-28', '2010-08-13', '2010-08-29', '2010-09-14', '2010-09-30', '2010-10-16', '2010-11-01', '2010-11-17', '2010-12-03', '2010-12-19', '2011-01-01', '2011-01-17', '2011-02-02', '2011-02-18', '2011-03-06', '2011-03-22', '2011-04-07', '2011-04-23', '2011-05-09', '2011-05-25', '2011-06-10', '2011-06-26', '2011-07-12', '2011-07-28', '2011-08-13', '2011-08-29', '2011-09-14', '2011-09-30', '2011-10-16', '2011-11-01', '2011-11-17', '2011-12-03', '2011-12-19', '2012-01-01', '2012-01-17', '2012-02-02', '2012-02-18', '2012-03-05', '2012-03-21', '2012-04-06', '2012-04-22', '2012-05-08', '2012-05-24', '2012-06-09', '2012-06-25', '2012-07-11', '2012-07-27', '2012-08-12', '2012-08-28', '2012-09-13', '2012-09-29', '2012-10-15', '2012-10-31', '2012-11-16', '2012-12-02', '2012-12-18', '2013-01-01', '2013-01-17', '2013-02-02', '2013-02-18', '2013-03-06', '2013-03-22', '2013-04-07', '2013-04-23', '2013-05-09', '2013-05-25', '2013-06-10', '2013-06-26', '2013-07-12', '2013-07-28', '2013-08-13', '2013-08-29', '2013-09-14', '2013-09-30', '2013-10-16', '2013-11-01', '2013-11-17', '2013-12-03', '2013-12-19', '2014-01-01', '2014-01-17', '2014-02-02', '2014-02-18', '2014-03-06', '2014-03-22', '2014-04-07', '2014-04-23', '2014-05-09', '2014-05-25', '2014-06-10', '2014-06-26', '2014-07-12', '2014-07-28', '2014-08-13', '2014-08-29', '2014-09-14', '2014-09-30', '2014-10-16', '2014-11-01', '2014-11-17', '2014-12-03', '2014-12-19', '2015-01-01', '2015-01-17', '2015-02-02', '2015-02-18', '2015-03-06', '2015-03-22', '2015-04-07', '2015-04-23', '2015-05-09', '2015-05-25', '2015-06-10', '2015-06-26', '2015-07-12', '2015-07-28', '2015-08-13', '2015-08-29', '2015-09-14', '2015-09-30', '2015-10-16', '2015-11-01', '2015-11-17', '2015-12-03', '2015-12-19', '2016-01-01', '2016-01-17', '2016-02-02', '2016-02-18', '2016-03-05', '2016-03-21', '2016-04-06', '2016-04-22', '2016-05-08', '2016-05-24', '2016-06-09', '2016-06-25', '2016-07-11', '2016-07-27', '2016-08-12', '2016-08-28', '2016-09-13', '2016-09-29', '2016-10-15', '2016-10-31', '2016-11-16', '2016-12-02', '2016-12-18', '2017-01-01', '2017-01-17', '2017-02-02', '2017-02-18', '2017-03-06', '2017-03-22', '2017-04-07', '2017-04-23', '2017-05-09', '2017-05-25', '2017-06-10', '2017-06-26', '2017-07-12', '2017-07-28', '2017-08-13', '2017-08-29', '2017-09-14', '2017-09-30', '2017-10-16', '2017-11-01', '2017-11-17', '2017-12-03', '2017-12-19', '2018-01-01', '2018-01-17', '2018-02-02', '2018-02-18', '2018-03-06', '2018-03-22', '2018-04-07', '2018-04-23', '2018-05-09', '2018-05-25', '2018-06-10', '2018-06-26', '2018-07-12', '2018-07-28', '2018-08-13', '2018-08-29', '2018-09-14', '2018-09-30', '2018-10-16', '2018-11-01', '2018-11-17', '2018-12-03', '2018-12-19', '2019-01-01', '2019-01-17', '2019-02-02', '2019-02-18', '2019-03-06', '2019-03-22', '2019-04-07', '2019-04-23', '2019-05-09', '2019-05-25', '2019-06-10', '2019-06-26', '2019-07-12', '2019-07-28', '2019-08-13', '2019-08-29', '2019-09-14', '2019-09-30', '2019-10-16', '2019-11-01', '2019-11-17', '2019-12-03', '2019-12-19', '2020-01-01', '2020-01-17', '2020-02-02', '2020-02-18', '2020-03-05', '2020-03-21', '2020-04-06', '2020-04-22', '2020-05-08', '2020-05-24', '2020-06-09', '2020-06-25', '2020-07-11', '2020-07-27']
And these are the dates I want in the data frame:
dates = ['2000-07-11', '2001-07-12', '2002-07-12', '2003-07-12', '2004-07-11',
'2005-07-12', '2006-07-12', '2007-07-12', '2008-07-11', '2009-07-12',
'2010-07-12', '2011-07-12', '2012-07-11', '2013-07-12', '2014-07-12',
'2015-07-12', '2016-07-11', '2017-07-12', '2018-07-12', '2019-07-12',
'2020-07-11']
I tried just defining a new dates data frame, but I think that caused problems for me later in the code, so I would like to just subset the first dates data frame if there is an easy way to do it.
Thank you for your help
Transform data to growth rates in Python
I have two variables and I want to express one of them (monetary_base) in terms of monthly growth.
How can I do that?. In the R language you should first transform the data into time series, in Python is this also the case?
#LLamando a las series que buscamos
inflacion = llamada_api('https://api.estadisticasbcra.com/inflacion_mensual_oficial')
base_monetaria = llamada_api('https://api.estadisticasbcra.com/base')
#Armando DataFrames
df = pd.DataFrame(inflacion)
df_bm = pd.DataFrame(base_monetaria)
#Renombrando columnas
df = df.rename(columns={'d':'Fecha',
'v':'IPC'})
df_bm = df_bm.rename(columns={'d':'Fecha',
'v':'base_monetaria'})
#Arreglando tipo de datos
df['Fecha']=pd.to_datetime(df['Fecha'])
df_bm['Fecha']=pd.to_datetime(df_bm['Fecha'])
#Verificando que las fechas esten en formato date
df['Fecha'].dtype
df_bm['Fecha'].dtype
#Filtrando
df_ipc = df[(df['Fecha'] > '2002-12-31')]
df_bm_filter = df_bm[(df_bm['Fecha'] > '2002-12-31')]
#Graficando
plt.figure(figsize=(14,12))
df_ipc.plot(x = 'Fecha', y = 'IPC')
plt.title('IPC-Mensual', fontdict={'fontsize':20})
plt.ylabel('IPC')
plt.xticks(rotation=45)
plt.show()
The data looks like this
Fecha base_monetaria
1748 2003-01-02 29302
1749 2003-01-03 29360
1750 2003-01-06 29524
1751 2003-01-07 29867
1752 2003-01-08 29957
... ...
5966 2020-02-18 1941302
5967 2020-02-19 1941904
5968 2020-02-20 1887975
5969 2020-02-21 1855477
5970 2020-02-26 1807042
The idea is to take the data for the last day of the month and calculate the growth rate with the data for the last day of the previous month.
You can try something like this
from pandas.tseries.offsets import MonthEnd
import pandas as pd
df = pd.DataFrame({'Fecha': ['2020-01-31', '2020-02-29', '2020-03-31', '2020-05-31', '2020-04-30', '2020-07-31', '2020-06-30', '2020-08-31', '2020-09-30', '2020-10-31', '2020-11-30', '2020-12-31'],
'price': ['32132', '54321', '3213121', '432123', '32132', '54321', '32132', '54321', '3213121', '432123', '32132', '54321']})
df['Fecha'] = df['Fecha'].astype('datetime64[ns]')
df['is_month_end'] = df['Fecha'].dt.is_month_end
df = df[df['is_month_end'] == True]
df.sort_values('Fecha',inplace=True)
df.reset_index(drop=True, inplace = True)
def change(x,y):
try:
index = df[df['Fecha']==y].index.item()
last = df.loc[index-1][1]
return float(x)/float(last)
except:
return 0
df['new_column'] = df.apply(lambda row: change(row['price'],row['Fecha']), axis=1)
df.head(12)
Assuming the base_moetaria is a monthly cumulative value then
df = pd.DataFrame({'Fecha': ['2020-01-31', '2020-02-29', '2020-03-31', '2020-05-31', '2020-04-30', '2020-07-31', '2020-06-30', '2020-08-31', '2020-09-30', '2020-10-31', '2020-11-30', '2020-12-31'],
'price': [32132, 54321, 3213121, 432123, 32132, 54321, 32132, 54321, 3213121, 432123, 32132, 54321]})
df['Fecha'] = pd.to_datetime(df['Fecha'])
df.set_index('Fecha', inplace=True)
new_df = df.groupby(pd.Grouper(freq="M")).tail(1).reset_index()
new_df['rate'] = (new_df['price'] -new_df['price'].shift(1))/new_df['price'].shift(1)
The new_df['rate'] will give you the growth rate the way you explained in the comment below
The problem can be solve creating a column with the lag values of base_monetaria df_bm_filter['is_month_end'] = df_bm_filter['Fecha'].dt.is_month_end df_last_date = df_bm_filter[df_bm_filter['is_month_end'] == True] df_last_date['base_monetaria_lag'] = df_last_date['base_monetaria'].shift(1) df_last_date['bm_growth'] = (df_last_date['base_monetaria'] - df_last_date['base_monetaria_lag']) / df_last_date['base_monetaria_lag']
How to filter two datetime indices?
I have two datetime indices - one being a date_range of business days and the other being a list of holidays.
I filter the holiday list by a start and end date. But now I need to join them and drop any duplicates (holidays and trading days both exist).
Finally I need to convert the daterange into a list of formatted strings ie: yyyy_mm_dd that I can iterate through later.
Here is my code so far:
import datetime
import pandas as pd
from pandas.tseries.holiday import AbstractHolidayCalendar, Holiday, nearest_workday, \
USMartinLutherKingJr, USPresidentsDay, GoodFriday, USMemorialDay, \
USLaborDay, USThanksgivingDay
class USTradingCalendar(AbstractHolidayCalendar):
rules = [
Holiday('NewYearsDay', month=1, day=1, observance=nearest_workday),
USMartinLutherKingJr,
USPresidentsDay,
GoodFriday,
USMemorialDay,
Holiday('USIndependenceDay', month=7, day=4, observance=nearest_workday),
USLaborDay,
USThanksgivingDay,
Holiday('Christmas', month=12, day=25, observance=nearest_workday)
]
def get_trading_close_holidays(year):
inst = USTradingCalendar()
return inst.holidays(datetime.datetime(year-1, 12, 31),
datetime.datetime(year, 12, 31))
start_date = "2017_07_01"
end_date = "2017_08_31"
start_date = datetime.datetime.strptime(start_date,"%Y_%m_%d").date()
end_date = datetime.datetime.strptime(end_date,"%Y_%m_%d").date()
date_range = pd.bdate_range(start = start_date, end = end_date, name =
"trading_days")
holidays = get_trading_close_holidays(start_date.year)
holidays = holidays.where((holidays.date > start_date) &
(holidays.date < end_date))
holidays = holidays.dropna(how = 'any')
date_range = date_range.where(~(date_range.trading_days.isin(holidays)))
Consider filtering by boolean condition:
date_range = date_range[date_range.date != holidays.date]
print(date_range) # ONE HOLIDAY 2017-07-04 DOES NOT APPEAR
# DatetimeIndex(['2017-07-03', '2017-07-05', '2017-07-06', '2017-07-07',
# '2017-07-10', '2017-07-11', '2017-07-12', '2017-07-13',
# '2017-07-14', '2017-07-17', '2017-07-18', '2017-07-19',
# '2017-07-20', '2017-07-21', '2017-07-24', '2017-07-25',
# '2017-07-26', '2017-07-27', '2017-07-28', '2017-07-31',
# '2017-08-01', '2017-08-02', '2017-08-03', '2017-08-04',
# '2017-08-07', '2017-08-08', '2017-08-09', '2017-08-10',
# '2017-08-11', '2017-08-14', '2017-08-15', '2017-08-16',
# '2017-08-17', '2017-08-18', '2017-08-21', '2017-08-22',
# '2017-08-23', '2017-08-24', '2017-08-25', '2017-08-28',
# '2017-08-29', '2017-08-30', '2017-08-31'],
# dtype='datetime64[ns]', name='trading_days', freq=None)
And using astype() to convert the datetime index to string type array, even tostring() for list conversion:
strdates = date_range.date.astype('str').tolist()
print(strdates)
# ['2017-07-03', '2017-07-05', '2017-07-06', '2017-07-07', '2017-07-10',
# '2017-07-11', '2017-07-12', '2017-07-13', '2017-07-14', '2017-07-17',
# '2017-07-18', '2017-07-19', '2017-07-20', '2017-07-21', '2017-07-24',
# '2017-07-25', '2017-07-26', '2017-07-27', '2017-07-28', '2017-07-31',
# '2017-08-01', '2017-08-02', '2017-08-03', '2017-08-04', '2017-08-07',
# '2017-08-08', '2017-08-09', '2017-08-10', '2017-08-11', '2017-08-14',
# '2017-08-15', '2017-08-16', '2017-08-17', '2017-08-18', '2017-08-21',
# '2017-08-22', '2017-08-23', '2017-08-24', '2017-08-25', '2017-08-28',
# '2017-08-29', '2017-08-30', '2017-08-31']