import pandas as pd
import numpy as np

temperature_Oslo = pd.Series([4, 6, 6, 9, 8, 7, 8]) # temperature in the middle of the day at Blindern for the next 7 days

temperature_Oslo

temperature_Oslo.index # same as range(7) or np.arange(7)

# indexes can be later modified by doing
temperature_Oslo.index = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
temperature_Oslo.index

ndata = {'Monday' : 4 , 'Tuesday' : 6 , 'Wednesday' : 6 , 'Thursday' : 9 , 'Friday' : 8 , 'Saturday' : 7, 'Sunday' : 8}


# feed pd.Series the previously created dictionary
temperature_Oslo = pd.Series(ndata)

# print the data on the screen
temperature_Oslo

# change in temperature between Monday and Tuesday
print (temperature_Oslo['Monday'] - temperature_Oslo['Tuesday'])

# similar to
print (temperature_Oslo.Monday - temperature_Oslo.Tuesday)

data = {'Day of the week' : ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'],
        'Temperature in the middle of the day' : [4, 6, 6, 9, 8, 7, 8],
        'Wind (m/s)' : [1, 6, 2, 2, 4, 3, 3],
        'Weather conditions' : ['Cloud & Sun', 'Cloud', 'Cloud', 'Cloud & Sun', 'Cloud & Sun', 'Sun', 'Cloud']}

frame = pd.DataFrame(data)

frame

frame.head() # print the first five rows

print (frame.columns) # print name of the columns

np.mean(frame['Temperature in the middle of the day'])

pd.DataFrame(data, columns=['Day of the week', 'Weather conditions', 'Temperature in the middle of the day', 'Wind (m/s)'])

print (frame.loc[0])

print (frame.loc[6])

# we create a new DataFrame with a new column
nframe = pd.DataFrame(data, columns=['Day of the week', 'Weather conditions', 'Temperature in the middle of the day', 
                                     'Wind (m/s)', 'Precipitation (mm)'])

nframe

nframe['Precipitation (mm)'].values[0] = 2.3

nframe

nframe['Precipitation (mm)'].values[1:4] = 1.0

nframe

nframe['Precipitation (mm)'] = 0.0 #equivalent to nframe['Precipitation (mm)'].values[:] = 0.0

nframe

nframe = nframe.set_index('Day of the week')

nframe

nframe.loc['Monday']

nframe.iloc[0]

# re-indexing
nframe2 = nframe.reindex(['Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday','Monday', 'Tuesday'])

nframe2

nframe2.sort_values(by='Temperature in the middle of the day') #default ascending

nframe2.sort_values(by='Temperature in the middle of the day', ascending=False) #descending

nframe2.sort_values(by=['Temperature in the middle of the day', 'Wind (m/s)'], ascending=False) #descending

path = '../data/Manon/' #directory to the file we would like to import
filename = 'data_manon_python.xlsx'  # filename

frame = pd.read_excel(path + filename)

frame.head()

import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec

%matplotlib inline

fig = plt.figure(figsize=(16,8))


gs=GridSpec(2,4) # 2 rows, 4 columns

ax1=fig.add_subplot(gs[:,:2]) # Span all rows, firs two columns
ax2=fig.add_subplot(gs[:,2]) # Span all rows, third column
ax3=fig.add_subplot(gs[:,3]) # Span all rows, fourth column

ax1.set_title('Calibrated Age')
ax2.set_title('Aluminium content')
ax3.set_title('Silicate content')

ax1.plot(frame['Age. Cal. CE'],frame['depth (m)'],"ko")
ax1.set_ylim(ax1.get_ylim()[::-1]) 

ax2.plot(frame['Al'],frame['depth (m)'],"bo")
ax2.set_ylim(ax2.get_ylim()[::-1]) 

ax3.plot(frame['Si'],frame['depth (m)'],"ro")
ax3.set_ylim(ax3.get_ylim()[::-1]) 

fig.tight_layout()

from datetime import datetime

now = datetime.now()

now

now.year, now.month, now.day

timestamp = datetime(2019, 1, 1) # date format

str(timestamp) # string format

timestamp.strftime('%Y-%m-%d') # string format

timestamp_str = '2019/04/01'

datetime.strptime(timestamp_str, '%Y/%m/%d')

from dateutil.parser import parse

parse('2019/04/01')

parse('01/04/2019', dayfirst=True) # day in front

pd.date_range?

dates_2019 = pd.date_range(start='2019/01/01', end='2019/04/01') # default is daily data

dates_2019

list_of_dates = ['01/01/2019', '19/01/2019', '25/02/2019', '07/03/2019', '01/04/2019'] # day-first european style


converted_list_of_dates = [parse(x, dayfirst=True) for x in list_of_dates]

converted_list_of_dates

frame['Age. Cal. CE'].head() #print the first 5 values

years = np.floor(frame['Age. Cal. CE'].values)

years

years = years.astype('int')

years_str = years.astype('str')

years_str

months_and_days = frame['Age. Cal. CE'].values - years

months_and_days

months_and_days2 = np.round(months_and_days * 365.25)
months_and_days3 = months_and_days2.astype('int') # integer
months_and_days3

# set 0 equal to 1 (avoid problem with day of the year = 0)
months_and_days3[months_and_days3 < 1] = 1 

months_and_days3

months_and_days_str = months_and_days3.astype('str')
months_and_days_str

tmp_date = years_str[0].zfill(4) + "/" + months_and_days_str[0].zfill(3) #zfill write 331 as 0331

tmp_date

datetime_viking = [] # create an empty list

# need to loop through the numpy array
for ix, dates_i in np.ndenumerate(months_and_days_str): #loop through strings
    
    # we save only the dates before Jesus
    if years[ix] > 0: #datetime does not support negative years or BC dates, need to work with Julian date format?
        tmp_date = years_str[ix].zfill(4) + "/" + months_and_days_str[ix].zfill(3)
        
        # save to originally empty list
        datetime_viking.append(datetime.strptime(tmp_date, '%Y/%j'))
        
datetime_viking[:5] # print first five dates

# need to convert to pandas datetime (datetime_viking is a list while we need to work with pandas or numpy array)

# empty panda series
date_pandas = pd.Series([])

#need to use period if we want to work with period younger than 1677

for ix, datev in enumerate(datetime_viking):
    
    date_tmp = pd.Period(year= datetime_viking[ix].year, month =datetime_viking[ix].month, day = datetime_viking[ix].day, freq='D')  
    #print (date_tmp)
    date_pandas = date_pandas.append(pd.Series([date_tmp]))
    
date_pandas.head()

# we need to create a new data frame containing only data before JC
frame_shortened = frame.iloc[0:len(datetime_viking)]

# and we set the dates as index
frame_shortened.index = date_pandas

# let's have a look at the dataframe
frame_shortened.head()

nframe_resampled = frame_shortened.resample('25Y').mean() # seems that it's indexing from year 0000 and to our time now
nframe_resampled.head()
#every 10 years
#frame_shortened.resample('10Y').mean().head()

# you can also sum the values
#frame_shortened.resample('Y').sum().head()
#frame_shortened.resample('Y', closed='right').sum().head()

frame_shortened.tail()

fig = plt.figure(figsize=(16,8))


gs=GridSpec(2,4) # 2 rows, 4 columns

ax1=fig.add_subplot(gs[:,:2]) # Span all rows, firs two columns
ax2=fig.add_subplot(gs[:,2]) # Span all rows, third column
ax3=fig.add_subplot(gs[:,3]) # Span all rows, fourth column

ax1.set_title('Calibrated Age')
ax2.set_title('Aluminium content')
ax3.set_title('Silicate content')

ax1.plot(nframe_resampled['Age. Cal. CE'],nframe_resampled['depth (m)'],"k-o")
ax1.set_xlim(ax1.get_xlim()[::-1]) 
ax1.set_ylim(ax1.get_ylim()[::-1]) 

ax2.plot(nframe_resampled['Al'],nframe_resampled['depth (m)'],"b-o")
ax2.set_xlim(ax2.get_xlim()[::-1]) 
ax2.set_ylim(ax1.get_ylim()[::-1]) 

ax3.plot(nframe_resampled['Si'],nframe_resampled['depth (m)'],"r-o")
ax3.set_xlim(ax3.get_xlim()[::-1]) 
ax3.set_ylim(ax3.get_ylim()[::-1]) 

fig.tight_layout()

import pathlib
import glob
import pandas as pd
import numpy as np
import os


path = '/opt/uio/deep_python/data/Nina/' #directory to the file we would like to import
filenames = glob.glob(path + "Inline_*.dat")

for file in filenames:
    print(file)
    data = np.loadtxt(file, dtype='str')
    frame=pd.DataFrame(data)
    frame[1].astype('float', inplace=True)
    frame[2].astype('float', inplace=True)
    frame[3] = frame[3].astype('float', inplace=True)+0.000001
    
    newfile = os.path.dirname(file) + '/' + pathlib.Path(file).stem + '_cal.dat'
    frame.to_csv(newfile, header=None, sep=' ', float_format="%.6f",  index=False)