I've been using PyEphem to play around with the motion of planets relative to a certain position on Earth. However, I notice sometimes that the results I get from PyEphem appear non-continuous for Declination, while Right Ascension appears continuous. The RA and Dec coordinates are taken in these plots every 30 minutes.
I would expect the stellar bodies to move in a continuous manner, however when declination is negative it looks discontinuous.
Any ideas why this would be? I can post my script if that helps as well.
Code:
from ephem import *
import datetime
import re
def cSTI(number):
degrees = int(number[0])
minutes = int(number[1])
seconds = float(number[2])
full = degrees + (minutes/60) + (seconds/60/60)
return round(full,2)
planets = [Sun(),Mercury(),Venus(),Moon(),Mars(),Jupiter(),Saturn(),Uranus(),Neptune(),Pluto()]
masses = [1.989*10**30,.330*10**24,4.87*10**24,0.073*10**24,0.642*10**24,1898*10**24,568*10**24,86.8*10**24,102*10**24,0.0146*10**24]
earthMass = 5.97*10**24
gravitational_constant = 6.67408 * 10**-11
theYear = '2018'
theMonth = '9'
theDay = '8'
revere = Observer()
revere.lon = '-42.4084'
revere.lat = '71.0120'
currentDate = datetime.datetime(2018,11,30,12,0,0)
lowerDateLimit = datetime.datetime(2018,9,1,12,0,0)
revere.date = currentDate
print('DATE SUN(RA) SUN(DEC) SUN(AZI) SUN(GFORCE) MERCURY(RA) MERCURY(DEC) MERCURY(AZI) MERCURY(GFORCE) VENUS(RA) VENUS(DEC) VENUS(AZI) VENUS(GFORCE) MOON(RA) MOON(DEC) MOON(AZI) MOON(GFORCE) MARS(RA) MARS(DEC) MARS(AZI) MARS(GFORCE) JUPITER(RA) JUPITER(DEC) JUPITER(AZI) JUPITER(GFORCE) SATURN(RA) SATURN(DEC) SATURN(AZI) SATURN(GFORCE) URANUS(RA) URANUS(DEC) URANUS(AZI) URANUS(GFORCE) NEPTUNE(RA) NEPTUNE(DEC) NEPTUNE(AZI) NEPTUNE(GFORCE) PLUTO(RA) PLUTO(DEC) PLUTO(AZI) PLUTO(GFORCE) ')
while (currentDate> lowerDateLimit):
print('%s ' % (revere.date),end = ' ')
planetIndex = 0;
for planet in planets:
planet.compute(revere)
rightascension = str(planet.ra)
declination = str(planet.dec)
azimuth = str(planet.az)
rightascension = re.split('[:]',rightascension)
declination = re.split('[:]',declination)
azimuth = re.split('[:]',azimuth )
rightascension = cSTI(rightascension);
declination = cSTI(declination);
azimuth = cSTI(azimuth);
GFORCE = gravitational_constant * ((masses[planetIndex]*earthMass)/(planet.earth_distance**2))
print('%s %s %s %s ' % (rightascension,declination,azimuth,GFORCE),end = ' ')
planetIndex+=1
print()
currentDate += datetime.timedelta(minutes=-30)
revere.date = currentDate
I believe the problem is in your manual conversion from ephem.Angle() (which is the object of ra, azi, etc.) to float.
EDIT
In particular, the problem arises because in your cSTI() function, when the value is negative, you should be subtracting (rather then adding) the different values.
A corrected implementation would look like:
import math
def cSTI(number):
degrees = int(number[0])
minutes = int(number[1])
seconds = float(number[2])
full = degrees + \
math.copysign((minutes / 60), degrees) + \
math.copysign((seconds / 60 / 60), degrees)
return round(full, 2)
Note that this is some sort of minimal modification to your code to make it work. I would suggest you to write some documents on how to write better code, starting from PEP8.
Also, you should avoid magic numbers like those wild 60 in your code.
If I were to do this manually, I would also avoid using unnecessary regular expressions, and would actually start from an ephem.Angle() object like:
import math
MIN_IN_DEG = SEC_IN_MIN = 60
def ephem_angle_to_float(angle):
degrees, minutes, seconds = [float(s) for s in str(angle).split(':')]
value = abs(degrees) + \
minutes / MIN_IN_DEG + \
seconds / SEC_IN_MIN / MIN_IN_DEG
return math.copysign(value, degrees)
and this could be called directly to planet.ra or planet.dec in your code.
But I would not do that manually. See below.
But the good news is that there is no need for manually computing that yourself, you could just cast that to float for the value in radians, as indicated in the official documentation.
The following code is a polished version of yours, which produces tabular data as a list of lists (which could be easily converted to CSV using csv from Python standard library).
A number of modification have been made:
celestial bodies and masses are more explicit now
the ephem objects are create dynamically
the G constant is now taken from scipy.constants (which is being fetched from latest CODATA)
the labels and the data is created dynamically
explicit indexing was avoided when possible
start and end dates have been somewhat inverted as it is irrelevant for the problem
Note that no conversion is performed here, as this would make us loose information.
Here is the code:
import numpy as np
import scipy as sp
import ephem
import scipy.constants
import datetime
celestial_masses = {
'sun': 1.989e30,
'mercury': 0.330e24,
'venus': 4.870e24,
'earth': 5.970e24,
'moon': 0.073e24,
'mars': 0.642e24,
'jupiter': 1898e24,
'saturn': 568e24,
'uranus': 86.8e24,
'neptune': 102e24,
'pluto': 0.0146e24, }
celestials = {
name: (eval('ephem.{}()'.format(name.title())), mass)
for name, mass in celestial_masses.items() if name != 'earth'}
gg = sp.constants.gravitational_constant
observer = ephem.Observer()
# Revere, Massachusetts, USA
observer.lon = '-42.4084'
observer.lat = '71.0120'
start_datetime = datetime.datetime(2018, 9, 1, 12, 0, 0)
end_datetime = datetime.datetime(2018, 11, 30, 12, 0, 0)
values = 'ra', 'dec', 'azi', 'gforce'
labels = ('date',) + tuple(
'{}({})'.format(name, value)
for name in celestials.keys()
for value in values)
data = []
observer.date = start_datetime
delta_datetime = datetime.timedelta(minutes=30)
while (observer.date.datetime() < end_datetime):
row = [observer.date]
for name, (body, mass) in celestials.items():
body.compute(observer)
row.extend([
body.ra, body.dec, body.az,
gg * ((mass * celestial_masses['earth']) / (body.earth_distance ** 2))])
data.append(row)
observer.date = observer.date.datetime() + delta_datetime
To convert to CSV (with float values for ra, dec, az and gforce) it is possible to do like this:
import csv
filepath = 'celestial_bodies_revere_MA.csv'
with open(filepath, 'w') as file_obj:
csv_obj = csv.writer(file_obj)
csv_obj.writerow(labels)
for row in data:
# : use default string conversion
# csv_obj.writerow(row)
# : a possible conversion to float for all but the `date`
csv_obj.writerow([
float(x) if i != labels.index('date') else x
for i, x in enumerate(row)])
Additionally, here is some code for the plots, showing that the issue with negative values is gone:
import matplotlib as mpl
import matplotlib.pyplot as plt
x = [row[labels.index('date')].datetime() for row in data]
fig, axs = plt.subplots(4, 1, figsize=(16, 26))
for i, value in enumerate(values):
pos = i
for name in celestials.keys():
y = [row[labels.index('{}({})'.format(name, value))] for row in data]
axs[pos].plot(x, y, label=name)
axs[pos].set_title(value)
axs[pos].legend(loc='center left', bbox_to_anchor=(1, 0.5))
fig.tight_layout()
which produces:
Related
I'm trying to calculate cape and cin in a specified layer using metpy's cape_cin and get_layer functions. Im doing this for a RAP vertical profile that I access from the NCDC server, for 3 different parcel types (ML, MU, and SB). I get this error ValueError: zero-size array to reduction operation minimum which has no identity when I try to compute the mu and sb profiles, but not for the ml profile...even though i calculate the layers in the exact same way:
First, I have to create vertical profiles of pressure, temp, dewpoint, and heights, from the RAP vertical profile:
from datetime import datetime, timedelta
from siphon.catalog import TDSCatalog
from siphon.ncss import NCSS
import numpy as np
from metpy.units import units
import cartopy.crs as ccrs
from metpy.calc import (dewpoint_from_relative_humidity, mixed_parcel, most_unstable_parcel, parcel_profile, pressure_to_height_std, lcl, height_to_pressure_std, get_layer, cape_cin)
year=2019
month=5
day=23
hour=0
cenlat = 34.91269
cenlon = -98.21048
time_start = datetime(year, month, day, hour, 0) #specified time
hour = time_start.hour
if hour < 10:
hour = '0'+str(hour)
day = time_start.day
if day < 10:
day = '0'+str(day)
month = time_start.month
if month < 10:
month = '0'+str(month)
cat = TDSCatalog('https://www.ncdc.noaa.gov/thredds/catalog/model-rap130-old/'+str(time_start.year)+str(month)+'/'+str(time_start.year)+str(month)+str(day)+'/catalog.html?dataset=rap130-old/'+str(time_start.year)+str(month)+'/'+str(time_start.year)+str(month)+str(day)+'/rap_130_'+str(time_start.year)+str(month)+str(day)+'_'+str(hour)+'00_000.grb2')
latest_ds = list(cat.datasets.values())[0]
print(latest_ds.access_urls)
ncss = NCSS(latest_ds.access_urls['NetcdfSubset'])
query = ncss.query()
query.variables('Pressure_surface').variables('Geopotential_height_isobaric').variables('Geopotential_height_surface').variables('Relative_humidity_isobaric').variables('Temperature_isobaric').variables('Dewpoint_temperature_height_above_ground').variables('Temperature_height_above_ground').variables
query.add_lonlat().lonlat_box(cenlon-2.1, cenlon +2.1, cenlat-2.1, cenlat+2.1)
data1 = ncss.get_data(query)
dlev = data1.variables['Geopotential_height_isobaric'].dimensions[1]
dlat = data1.variables['Geopotential_height_isobaric'].dimensions[2]
dlon = data1.variables['Geopotential_height_isobaric'].dimensions[3]
SFCP = (np.asarray(data1.variables['Pressure_surface'][:])/100.) * units('hPa')
hgt = np.asarray(data1.variables['Geopotential_height_isobaric'][:]) * units('meter')
sfc_hgt = np.asarray(data1.variables['Geopotential_height_surface'][:]) * units('meter')
Temp_up = np.asarray(data1.variables['Temperature_isobaric'][:]) * units('kelvin')
RH_up = np.asarray(data1.variables['Relative_humidity_isobaric'][:])
Td = (np.asarray(data1.variables['Dewpoint_temperature_height_above_ground'][:]) * units('kelvin')).to('degC')
T = np.asarray(data1.variables['Temperature_height_above_ground'][:]) * units('kelvin')
# Get the dimension data
lats_r = data1.variables[dlat][:]
lons_r= data1.variables[dlon][:]
lev = (np.asarray(data1.variables[dlev][:])/100.) * units('hPa')
# Set up our array of latitude and longitude values and transform to the desired projection.
flon = float(cenlon)
flat = float(cenlat)
crs = ccrs.PlateCarree()
crlons, crlats = np.meshgrid(lons_r[:]*1000, lats_r[:]*1000)
trlatlons = crs.transform_points(ccrs.LambertConformal(central_longitude=265, central_latitude=25, standard_parallels=(25.,25.)),crlons,crlats)
trlons = trlatlons[:,:,0]
trlats = trlatlons[:,:,1]
dlon = np.abs(trlons - cenlon)
dlat = np.abs(trlats - cenlat)
ilon = np.where(dlon == np.min(dlon)) #position in the dlon array with minimal difference between gridpoint lon and input lon
ilat = np.where(dlat == np.min(dlat)) #position in the dlat array with minimal difference between gridpoint lat and input lat
Tdc_up = dewpoint_from_relative_humidity(Temp_up[0,:,ilat[0][0], ilon[1][0]],RH_up[0,:,ilat[0][0], ilon[1][0]]/100)
p_sounding = np.sort(np.append(lev, SFCP[0,ilat[0][0], ilon[1][0]]))
ind = np.where(p_sounding >= SFCP[0,ilat[0][0], ilon[1][0]])[0][0]
hgt_sounding = np.insert(hgt[0,:,ilat[0][0], ilon[1][0]].magnitude, ind, sfc_hgt[0,ilat[0][0], ilon[1][0]].magnitude) * hgt.units
T_sounding = (np.insert(Temp_up[0,:,ilat[0][0], ilon[1][0]].magnitude, ind, T[0,0,ilat[0][0], ilon[1][0]].magnitude) * T.units).to(Tdc_up.units)
Td_sounding = np.insert(Tdc_up.magnitude, ind, Td[0,0,ilat[0][0], ilon[1][0]].magnitude) * Tdc_up.units
p_skewt = p_sounding[p_sounding <= SFCP[0,ilat[0][0], ilon[1][0]]]
hgt_skewt = hgt_sounding[p_sounding <= SFCP[0,ilat[0][0], ilon[1][0]]]
T_skewt = T_sounding[p_sounding <= SFCP[0,ilat[0][0], ilon[1][0]]]
Td_skewt = Td_sounding[p_sounding <= SFCP[0,ilat[0][0], ilon[1][0]]]
AGLhgts = hgt_skewt[::-1]-hgt_skewt[-1]
Next, I create vertical profiles for each parcel type using mixed_parcel, most_unstable, and parcel_profile functions, and compute the pressure values for th etop and bottom of the layer I want to calculate cape_cin for (the LCL to LCL+2km):
ml_p, ml_T, ml_Td = mixed_parcel(np.flip(p_skewt), np.flip(T_skewt), np.flip(Td_skewt))
ml_profile = parcel_profile(p_skewt[::-1], ml_T, ml_Td)
ml_profile = (ml_profile - 273.15*units('kelvin')).magnitude*units('degC')
mu_p, mu_T, mu_Td, mu_index = most_unstable_parcel(np.flip(p_skewt), np.flip(T_skewt), np.flip(Td_skewt))
mu_profile = parcel_profile(p_skewt[::-1], mu_T, mu_Td)
mu_profile = (mu_profile - 273.15*units('kelvin')).magnitude*units('degC')
#Note: sbpcl_profile will have the exact same values of p_skewt, T_skewt, and Td_skewt in pprof below:
pprof = parcel_profile(p_skewt[::-1], T_skewt[-1], Td_skewt[-1])
pprof = (pprof - 273.15*units('kelvin')).magnitude*units('degC')
mllcl = lcl(ml_p, ml_T, ml_Td)
mllcl_h = pressure_to_height_std(mllcl[0]) - hgt_skewt[-1]
mulcl = lcl(mu_p, mu_T, mu_Td)
mulcl_h = pressure_to_height_std(mulcl[0]) - hgt_skewt[-1]
sblcl = lcl(p_skewt[-1], T_skewt[-1], Td_skewt[-1])
sblcl_h = pressure_to_height_std(sblcl[0]) - hgt_skewt[-1]
mllcl2000 = mllcl_h + 2*units('kilometer')
mulcl2000 = mulcl_h + 2*units('kilometer')
sblcl2000 = sblcl_h + 2*units('kilometer')
mllcl2000_p = height_to_pressure_std(mllcl2000)
mulcl2000_p = height_to_pressure_std(mulcl2000)
sblcl2000_p = height_to_pressure_std(sblcl2000)
With all of that computed, I use the get_layer function to create the arrays of pressure, temp, dewpoint, and parcel temp I need to compute cape_cin, and then go to compute the actual cape_cin values in the layer of interest:
ml_LCL_CAPE_layer = get_layer(p_skewt, T_skewt, Td_skewt, ml_profile[::-1], bottom = mllcl[0], depth = mllcl[0] - mllcl2000_p)
mu_LCL_CAPE_layer = get_layer(p_skewt, T_skewt, Td_skewt, mu_profile[::-1], bottom = mulcl[0], depth = mulcl[0] - mulcl2000_p)
sb_LCL_CAPE_layer = get_layer(p_skewt, T_skewt, Td_skewt, pprof[::-1], bottom = sblcl[0], depth = sblcl[0] - sblcl2000_p)
mlLCLCAPE = cape_cin(ml_LCL_CAPE_layer[0], ml_LCL_CAPE_layer[1], ml_LCL_CAPE_layer[2], ml_LCL_CAPE_layer[3])
muLCLCAPE = cape_cin(mu_LCL_CAPE_layer[0], mu_LCL_CAPE_layer[1], mu_LCL_CAPE_layer[2], mu_LCL_CAPE_layer[3])
sbLCLCAPE = cape_cin(sb_LCL_CAPE_layer[0], sb_LCL_CAPE_layer[1], sb_LCL_CAPE_layer[2], sb_LCL_CAPE_layer[3])
mlLCLCAPEcin = mlLCLCAPE[0] + mlLCLCAPE[1]
muLCLCAPEcin = muLCLCAPE[0] + muLCLCAPE[1]
sbLCLCAPEcin = sbLCLCAPE[0] + sbLCLCAPE[1]
The arrays for pressure, temp, dewpoint, and parcel temp for each of the 3 get_layer functions appear to be populated with the correct values, and these 4 arrays for each parcel type are all the same shape. The mlLCLCAPEcin calculation above gives the correct output (99.26 j/kg - which verifies when I plot it on a SkewT), but the exact same calculation for the MU and SB profiles give the error referenced above. I'm using Metpy v 1.1, and have tried to use a different location and output from a different forecast hour, and still run into the same issue.
If I fix up your example code above (there were some name issues and some indexing issues) with:
T_sounding = (np.insert(Temp_up[0,:,ilat[0][0], ilon[1][0]].magnitude, ind, Temp_up[0,0,ilat[0][0], ilon[1][0]].magnitude) * Temp_up.units).to(Tdc_up.units)
Td_sounding = np.insert(Tdc_up.magnitude, ind, Tdc_up[0].magnitude) * Tdc_up.units
I don't get any error running right now. If my fix above doesn't fix it for you, it would be helpful to know if you're running the latest MetPy 1.1.
I'm new to python, so I'm sorry if I make any beginner mistakes. I'm trying to insert my text file into a netcdf.
I'm using the netcdf4 package and follow the example in this website: https://pyhogs.github.io/intro_netcdf4.html and I managed to reproduce the example (the example uses random data):
Problem: My text file contains: Lon, Lat , SST and when I try to insert this values, the netcdf file is created, however, it's not correct:
In my code I'm trying to apply a Barnes interpolation (var) or a griddata interpolation (interp).
I think this is what has to enter in my variable netcdf (maybe I'm wrong).
Here my code so far:
import os
import numpy as np
from scipy.interpolate import griddata
import matplotlib.pyplot as plt
import numpy.ma as ma
import netCDF4 as nc4
from numpy.random import uniform, seed
from metpy.interpolate import (interpolate_to_grid, remove_nan_observations, inverse_distance_to_grid, remove_repeat_coordinates)
# Open file
arq_sst = np.loadtxt(fname = "C:\\Users\\Rodrigo\\XYZ.txt", skiprows=0, delimiter=",")
# Getting the Arrays
lonf = arq_sst[:, 0]
latf = arq_sst[:, 1]
sstf = arq_sst[:, 2]
# Atmosphere level
z = [1]
#shapping grid
x_1, y_1 = np.meshgrid(lonf, latf)
#Barnes Interpolation
var = inverse_distance_to_grid(lonf, latf, sstf, x_1, y_1, r=100000, gamma=0.25, kappa=5.052, min_neighbors=3, kind='barnes')
#Or
#Another interpolation
interp = griddata((lonf, latf), sstf, (lonf[None,:], latf[:,None]), method='nearest')
#Open netcdf to write
f = nc4.Dataset('file_created.nc','w', format='NETCDF4')
#Creating group in netcdf file
tempgrp = f.createGroup('SAT_DATA')
#Specifying dimensions
tempgrp.createDimension('lon', len(lonf))
tempgrp.createDimension('lat', len(latf))
tempgrp.createDimension('z', len(z))
tempgrp.createDimension('time', None)
#Building variables
longitude = tempgrp.createVariable('Longitude', 'f4', 'lon')
latitude = tempgrp.createVariable('Latitude', 'f4', 'lat')
levels = tempgrp.createVariable('Levels', 'i4', 'z')
sst = tempgrp.createVariable('sst', 'f4', ('time', 'lon', 'lat', 'z'))
time = tempgrp.createVariable('Time', 'i4', 'time')
#Passing data into variables
longitude[:] = lonf
latitude[:] = latf
levels[:] = z
sst[0,:,:,:] = var
#get time in days since Jan 01,01
from datetime import datetime
today = datetime.today()
time_num = today.toordinal()
time[0] = time_num
#Add global attributes
f.description = "XYZ dataset containing one group"
f.history = "Created " + today.strftime("%d/%m/%y")
#Add local attributes to variable instances
longitude.units = 'degrees east'
latitude.units = 'degrees north'
time.units = 'days since Jan 01, 0001'
sst.units = 'degrees'
levels.units = 'meters'
sst.warning = 'This data is not real!'
#Closing the dataset
f.close()
Here is my text data(Header: Longitude,Latitude,SST). I decreased the number of lines to fit here:
-42.1870,-22.9940,22.4844
-37.4000,-29.9700,20.2000
-37.4200,-29.9600,20.1000
-39.1800,-30.0000,20.5000
-39.2100,-30.0000,20.4000
-39.2300,-30.0000,20.4000
-39.2200,-29.9800,20.4000
-39.2300,-29.9900,20.4000
-39.2000,-29.9800,20.4000
-39.1900,-30.0000,20.5000
-39.2800,-29.9900,20.5000
-39.2700,-29.9900,20.4000
-39.3400,-29.9700,20.5000
-39.3300,-29.9600,20.4000
-39.3100,-29.9600,20.4000
-39.3600,-29.9700,20.6000
-39.3500,-29.9900,20.4000
-39.3900,-29.9900,20.4000
-38.4600,-30.0000,20.3000
-38.4900,-29.9800,20.7000
-37.4800,-29.8800,20.4000
-37.5000,-29.8600,20.3000
-37.4600,-29.8900,20.3000
-41.3800,-29.9900,20.0000
-41.4000,-29.9900,20.1000
-41.0400,-29.9300,20.1000
-41.0200,-29.9200,20.2000
-41.0600,-29.9300,20.1000
-41.1000,-29.9400,19.9000
-41.0900,-29.9600,19.9000
-41.1100,-29.9800,19.9000
-41.1100,-29.9600,20.0000
-41.1200,-29.9400,20.0000
-41.1400,-29.9400,20.0000
-41.1600,-29.9500,20.1000
-41.1700,-29.9500,20.1000
-41.1900,-29.9700,20.0000
-41.1900,-29.9500,20.1000
-40.6800,-29.9900,20.1000
-40.7400,-29.9600,20.1000
-40.7700,-29.9700,20.1000
-40.7800,-29.9700,20.1000
-40.7100,-29.9000,20.1000
-40.7600,-29.9100,20.1000
-40.7400,-29.9000,20.1000
-40.7200,-29.9000,20.2000
-40.7600,-29.9200,20.1000
-40.7500,-29.9400,20.1000
-40.7800,-29.9100,20.2000
-40.8000,-29.9100,20.2000
-40.8100,-29.9300,20.1000
-40.8200,-29.9200,20.2000
-40.7900,-29.9300,20.2000
-40.7900,-29.9500,20.1000
-40.7700,-29.9300,20.1000
-40.8400,-29.9600,20.2000
-40.8600,-29.9600,20.3000
-40.9000,-29.9100,20.1000
-40.9100,-29.9100,20.0000
-40.3900,-29.9400,20.0000
-40.3900,-29.9200,20.0000
-40.4100,-29.9200,20.0000
-40.4100,-29.9400,20.0000
-40.3800,-29.9000,20.0000
-40.3800,-29.9200,20.0000
-40.4000,-29.9000,20.1000
-40.3700,-29.9600,20.0000
-40.3600,-29.9700,20.0000
-40.3800,-29.9800,20.0000
-40.4200,-29.9000,20.0000
-40.4300,-29.9300,20.1000
-40.4500,-29.9300,20.1000
-40.4700,-29.9300,20.0000
-40.4400,-29.9100,20.0000
-40.4500,-29.9100,20.0000
-40.4700,-29.9100,20.0000
-40.5000,-29.9400,19.9000
-40.5300,-29.9200,20.1000
-40.5100,-29.9200,20.1000
-40.4900,-29.9400,19.9000
-40.4900,-29.9200,20.0000
-40.6200,-30.0000,20.2000
-40.6000,-30.0000,20.1000
-40.6800,-29.9900,20.1000
-40.4000,-29.8400,20.1000
-40.4800,-29.8700,20.1000
-40.4500,-29.8300,20.3000
-40.4600,-29.8900,20.1000
-40.4600,-29.8700,20.0000
-40.5000,-29.8800,20.3000
-40.4900,-29.9000,20.1000
-40.5100,-29.9000,20.3000
-40.5300,-29.9000,20.2000
-40.5600,-29.8500,20.3000
-40.5800,-29.8500,20.3000
-40.6300,-29.9000,19.9000
-40.7100,-29.9000,20.1000
-40.0500,-29.9600,20.3000
-40.1100,-29.9800,20.2000
-40.1100,-30.0000,20.2000
Can anybody help me?
So there are a couple of things. First of all, you are not providing the correct equally spaced dimensions for the interpolation and the resulting netCDF file. This is how I created the space for the meshgrid, (I chose a linear space of 100 but depending on what resolution you want your data you may want to change this to whatever suits your purpose):
spacing_x = np.linspace(np.min(lonf),np.max(lonf),100)
spacing_y = np.linspace(np.min(latf),np.max(latf),100)
x_1, y_1 = np.meshgrid(spacing_x, spacing_y)
Then doing the interpolation as follows:
#Barnes Interpolation
var = inverse_distance_to_grid(lonf, latf, sstf, x_1, y_1, r=100000, gamma=0.25, kappa=5.052, min_neighbors=3, kind='barnes')
#Or
#Another interpolation
interp = griddata((lonf, latf), sstf, (x_1, y_1), method='nearest')
Finally you will want to add the linear spaces as the latitude and longitude dimensions since the interpolated data is being broadcasted to them:
#Passing data into variables
longitude[:] = x_1[0]
latitude[:] = y_1[:,0]
Another note is that for Panoply or other software to show your data in a Geo2D format, you will want to name your lat lon dimensions the same as your variables. The full code is below:
import os
import numpy as np
from scipy.interpolate import griddata
import matplotlib.pyplot as plt
import numpy.ma as ma
import netCDF4 as nc4
from numpy.random import uniform, seed
from metpy.interpolate import (interpolate_to_grid, remove_nan_observations, inverse_distance_to_grid, remove_repeat_coordinates)
# Open file
arq_sst = np.loadtxt(fname = r"C:\Users\Rodrigo\XYZ.txt", skiprows=0, delimiter=",")
# Getting the Arrays
lonf = arq_sst[:, 0]
latf = arq_sst[:, 1]
sstf = arq_sst[:, 2]
# Atmosphere level
z = [1]
#shapping grid
spacing_x = np.linspace(np.min(lonf),np.max(lonf),100)
spacing_y = np.linspace(np.min(latf),np.max(latf),100)
x_1, y_1 = np.meshgrid(spacing_x, spacing_y)
#Barnes Interpolation
var = inverse_distance_to_grid(lonf, latf, sstf, x_1, y_1, r=100000, gamma=0.25, kappa=5.052, min_neighbors=3, kind='barnes')
#Or
#Another interpolation
interp = griddata((lonf, latf), sstf, (x_1, y_1), method='nearest')
#Open netcdf to write
f = nc4.Dataset('file_created.nc','w', format='NETCDF4')
#Creating group in netcdf file
tempgrp = f.createGroup('SAT_DATA')
#Specifying dimensions
tempgrp.createDimension('longitude', len(spacing_x))
tempgrp.createDimension('latitude', len(spacing_y))
tempgrp.createDimension('z', len(z))
tempgrp.createDimension('time', None)
#Building variables
longitude = tempgrp.createVariable('longitude', 'f8', 'longitude', fill_value=np.nan)
latitude = tempgrp.createVariable('latitude', 'f8', 'latitude', fill_value=np.nan)
levels = tempgrp.createVariable('z', 'i4', 'z')
sst = tempgrp.createVariable('sst', 'f8', ('time','longitude','latitude','z'), fill_value=np.nan)
time = tempgrp.createVariable('time', 'f8', 'time', fill_value=np.nan)
#Passing data into variables
longitude[:] = x_1[0]
latitude[:] = y_1[:,0]
levels[:] = z
sst[0,:,:,:] = var
#get time in days since Jan 01,01
from datetime import datetime
today = datetime.today()
time_num = today.toordinal()
time[0] = time_num
#Add global attributes
f.description = "XYZ dataset containing one group"
f.history = "Created " + today.strftime("%d/%m/%y")
#Add local attributes to variable instances
longitude.units = 'degrees_east'
longitude.point_spacing = "even";
longitude._CoordinateAxisType = "Lon";
latitude.units = 'degrees_north'
latitude.point_spacing = "even";
latitude._CoordinateAxisType = "Lat";
time.units = "days since Jan 01, 0001";
time._ChunkSizes = [1]
sst.long_name = "SEA SURFACE TEMPERATURE"
sst.history = "From coads_climatology"
sst.units = "Deg C";
sst.missing_value = -1.0
sst._ChunkSizes = [1, 100, 100]
levels.units = 'meters'
sst.warning = 'This data is not real!'
#Closing the dataset
f.close()
Let me know if you have any questions.
I am trying to get stock data for a company and predict stock prices in the future. I know this isn't accurate, but I am using it as a learning tool. When using today's date as the end date and the predicted date as a date in the future my code appears to work. However, when using a past date and predicting the future this produces an error:
"ValueError: x and y must have same first dimension, but have shapes (220,) and (221,)"
I want to do this as then I would be able to compare predictions to actual prices.
import numpy as np
import datetime
import pandas_datareader as web
import statistics
import matplotlib.pyplot as plt
import pandas as pd
from matplotlib import style
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
stock_name = 'BP.L'
prices = web.DataReader(stock_name, 'yahoo', start = '2019-01-01', end = '2019-11-05').reset_index(drop = False)[['Date', 'Adj Close']]
#plt.plot(prices['Date'], prices['Adj Close'])
#plt.xlabel('Days')
#plt.ylabel('Stock Prices')
#plt.show()
# Parameter Definitions
# So : initial stock price
# dt : time increment -> a day in our case
# T : length of the prediction time horizon(how many time points to predict, same unit with dt(days))
# N : number of time points in the prediction time horizon -> T/dt
# t : array for time points in the prediction time horizon [1, 2, 3, .. , N]
# mu : mean of historical daily returns
# sigma : standard deviation of historical daily returns
# b : array for brownian increments
# W : array for brownian path
start_date = '2018-01-01'
end_date = '2019-01-01'
pred_end_date = '2019-11-05'
# We get daily closing stock prices
S_eon = web.DataReader(stock_name, 'yahoo', start_date, end_date).reset_index(drop = False)[['Date', 'Adj Close']]
So = S_eon.loc[S_eon.shape[0] -1, "Adj Close"]
dt = 1
n_of_wkdays = pd.date_range(start = pd.to_datetime(end_date,
format = "%Y-%m-%d") + pd.Timedelta('1 days'),
end = pd.to_datetime(pred_end_date,
format = "%Y-%m-%d")).to_series(
).map(lambda x:
1 if x.isoweekday() in range(1,6) else 0).sum()
T = n_of_wkdays
N = T / dt
t = np.arange(1, int(N) + 1)
returns = (S_eon.loc[1:, 'Adj Close'] - \
S_eon.shift(1).loc[1:, 'Adj Close']) / \
S_eon.shift(1).loc[1:, 'Adj Close']
mu = np.mean(returns)
sigma = np.std(returns)
scen_size = 10000
b = {str(scen): np.random.normal(0, 1, int(N)) for scen in range(1, scen_size + 1)}
W = {str(scen): b[str(scen)].cumsum() for scen in range(1, scen_size + 1)}
drift = (mu - 0.5 * sigma**2) * t
diffusion = {str(scen): sigma * W[str(scen)] for scen in range(1, scen_size + 1)}
S = np.array([So * np.exp(drift + diffusion[str(scen)]) for scen in range(1, scen_size + 1)])
S = np.hstack((np.array([[So] for scen in range(scen_size)]), S))
S_avg = np.mean(S)
print(S_avg)
#Plotting
plt.figure(figsize = (20,10))
for i in range(scen_size):
plt.title("Daily Volatility: " + str(sigma))
plt.plot(pd.date_range(start = S_eon["Date"].max(),
end = pred_end_date, freq = 'D').map(lambda x:
x if x.isoweekday() in range(1, 6) else np.nan).dropna(), S[i, :])
plt.ylabel('Stock Prices, €')
plt.xlabel('Prediction Days')
plt.show()
The error shows:
"File "C:\Users\User\Anaconda3\lib\site-packages\matplotlib\axes_base.py", line 270, in _xy_from_xy
"have shapes {} and {}".format(x.shape, y.shape))"
Could you try to add one day more to the prediction end date?
pred_end_date = '2019-11-06'
Your error is just a shape mismatch and your date series miss only one value
According to the documentation:
date.isoweekday()
Return the day of the week as an integer, where
Monday is 1 and Sunday is 7. For example, date(2002, 12,
4).isoweekday() == 3, a Wednesday. See also weekday(), isocalendar().
This returns a number between 1 and 7, and you're checking the range 1 to 6, converting other values to na. Then you dropna them, so you lose a value.
Change it to x if x.isoweekday() in range(1, 7) and it should work.
I changed the following and it now works:
"x if x.isoweekday() in range(1, 6) else np.nan).dropna(), S[i, :])"
to:
"x if x.isoweekday() in range(1, 6) else np.nan).dropna(), S[i, :-1])"
I'm jumping right into ArcPy to help out a friend who needs to automate some ArcGIS work. The problem at hand can be reduced to the following pseudocode:
accumulationRaster = SomeZeroedRaster
for each Raster pair, R, S, pertaining to rain and snow:
accumulationRaster += Con(S == 0, Con(R >= 1, 1, 0), 0)
In other words, iterate through all data of rain and snow of each day. Perform a logical operation on each cell pair, so that the corresponding cell in the accumulationRaster gets incremented if no snow (S == 0) and some rain (R >= 1).
Having no luck so far with various attempts, I thought I'd ask here, as the problem should have an easy solution. Full working code with the missing gaps pointed out below. Most of it is just trivial file path set up, and the above pseudocode should suffice:
The missing pieces are numbered 1, 2, and 3.
How to initializase a zero raster w/proper dimensions.
How to make a copy of this zero raster. If 1 is simple, just do this instead.
Increment accumulation raster.
import os
import arcpy
from arcpy import env
from arcpy.sa import *
in_workspace = "C:/Data/ProjectFolder"
os.chdir(in_workspace)
arcpy.env.workspace = in_workspace
maaneder = ["01_januar", "02_februar", "03_mars", "04_april",
"05_mai", "06_juni", "07_juli", "08_august",
"09_september", "10_oktober", "11_november", "12_desember"]
################ 1 ###################
baseAccum = #1: ?? (zero Raster w/correct type & dimensions)
######################################
outputPath = "rainDaysInGrowthSeason/"
for year in range(1997,2015):
for month in range(1, 13):
################ 2 ###################
monthAccum = #2: ?? (Copy of baseSum)
######################################
for day in range(1,32):
# e.g 1997/09_september/
fpath = str(year) + "/" + maaneder[month-1] + "/"
# e.g. 1997_09_08.asc
filesuffix = str(year) + "_" + str(month).zfill(2) + "_" + str(day).zfill(2) + ".asc"
# e.g. Snoedybde/1997/09_september/sd_1997_09_08.asc
snoefile = "Snoedybde/" + fpath + "sd_" + filesuffix
# e.g. rrl/1997/09_september/rrl_1997_09_08.asc
rrlfile = "rrl/" + fpath + "rrl_" + filesuffix
hasSnoe = os.path.isfile(snoefile)
hasRrl = os.path.isfile(rrlfile)
if (not hasSnoe or not hasRrl):
continue
snoeRaster = Raster(snoefile)
rrlRaster = Raster(rrlfile)
############## 3 #####################
monthAccum = #3: ??
# something a'la:
# += Con(snoeRaster == 0, Con(rrlRaster >= 1, 1, 0), 0)
######################################
# e.g. rainDaysInGrowthSeason/1997
outputDir = outputPath + str(year)
if (not os.path.isdir(outputDir)):
os.mkdir(outputDir)
monthAccum.save(outputDir + "/r_" + str(year) + "_" + str(month).zfill(2))
You can use arcpy.NumPyArrayToRaster to make a zero base raster.
if you wanted the raster to be 50 rows and 50 columns with a 1 meter resolution. you can do this.
import numpy
myArr = numpy.zeros(50, 50)
myZeroRaster = arcpy.NumPyArrayToRaster(myArr, arcpy.Point(355355, 3199277), 1, 1)
You can also convert a raster to a numpy array and then do your accumulations using one numpy array to another.
Or if you have Spatial analyst, you could use the tool create constant raster to create a zero based raster and then use the math -> Plus tool to add your rasters together
I wish to generate this in python:
http://classes.yale.edu/fractals/RandFrac/Market/TradingTime/Example1/Example1.html
but I'm incredibly stuck and new to this concept. Does anybody know of a library or gist for this?
Edit:
From what I can understand is that you need to split the fractal in 2 every time. So you have to calculate the y-axis point from the line between the two middle points. Then the two sections need to be formed according to the fractal?
Not 100% sure what you are asking, but as I understood from your comments, you want to generate a realistically looking stock market curve using the recursion described in the link.
As far as I understood the description in the linked page and some of the parent pages, it works like this:
You are given a start and an end point and a number of turning points in the form (t1, v1), (t2, v2), etc., for example start=(0,0), end=(1,1), turns = [(1/4, 1/2), (3/4, 1/4)], where ti and vi are fractions between 0 and 1.
You determine the actual turning points scaled to that interval between start and end and calculate the differences between those points, i.e. how far to go from pi to reach pi+1.
You shuffle those segments to introduce some randomness; when put together, they still cover exactly the same distance, i.e. they connect the original start and end point.
Repeat by recursively calling the function for the different segments between the new points.
Here's some Python code I just put together:
from __future__ import division
from random import shuffle
def make_graph(depth, graph, start, end, turns):
# add points to graph
graph.add(start)
graph.add(end)
if depth > 0:
# unpack input values
fromtime, fromvalue = start
totime, tovalue = end
# calcualte differences between points
diffs = []
last_time, last_val = fromtime, fromvalue
for t, v in turns:
new_time = fromtime + (totime - fromtime) * t
new_val = fromvalue + (tovalue - fromvalue) * v
diffs.append((new_time - last_time, new_val - last_val))
last_time, last_val = new_time, new_val
# add 'brownian motion' by reordering the segments
shuffle(diffs)
# calculate actual intermediate points and recurse
last = start
for segment in diffs:
p = last[0] + segment[0], last[1] + segment[1]
make_graph(depth - 1, graph, last, p, turns)
last = p
make_graph(depth - 1, graph, last, end, turns)
from matplotlib import pyplot
depth = 8
graph = set()
make_graph(depth, graph, (0, 0), (1, 1), [(1/9, 2/3), (5/9, 1/3)])
pyplot.plot(*zip(*sorted(graph)))
pyplot.show()
And here some example output:
I had a similar interest and developed a python3 library to do just what you want.
pip install fractalmarkets
See https://github.com/hyperstripe50/fractal-market-analysis/blob/master/README.md
Using #tobias_k solution and pandas, we can translate and scale the normalized fractal to a time-based one.
import arrow
import pandas as pd
import time
depth = 5
# the "geometry" of fractal
turns = [
(1 / 9, 0.60),
(5 / 9, 0.30),
(8 / 9, 0.70),
]
# select start / end time
t0 = arrow.now().floor("hours")
t1 = t0.shift(days=5)
start = (pd.to_datetime(t0._datetime), 1000)
end = (pd.to_datetime(t1._datetime), 2000)
# create a non-dimensionalized [0,0]x[1,1] Fractal
_start, _end = (0, 0), (1, 1)
graph = set()
make_graph(depth, graph, _start, _end, turns)
# just check graph length
assert len(graph) == (len(turns) + 1) ** depth + 1
# create a pandas dataframe from the normalized Fractal
df = pd.DataFrame(graph)
df.sort_values(0, inplace=True)
df.reset_index(drop=True, inplace=True)
# translate to real coordinates
X = pd.DataFrame(
data=[(start[0].timestamp(), start[1]), (end[0].timestamp(), end[1])]
).T
delta = X[1] - X[0]
Y = df.mul(delta) + X[0]
Y[0] = [*map(lambda x: pd.to_datetime(x, unit="s"), Y[0])]
# now resample and interpolate data according to *grid* size
grid ="min"
Z = Y.set_index(0)
A = Z.resample(grid).mean().interpolate()
# plot both graph to check errors
import matplotlib.pyplot as plt
ax = Z.plot()
A.plot(ax=ax)
plt.show()
showing both graphs:
and zooming to see interpolation and snap-to-grid differences: