import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
plt.style.use('fivethirtyeight')

# load in the data set; explore salaries of different job titles in San Francisco. 
salary_ranges = pd.read_csv('./data/Salary_Ranges_by_Job_Classification.csv')

# view the first few rows and the headers
salary_ranges.head()

# get...
# 1. how many rows of data there are
# 2. if there are any missing values
# 3. what data type each column has
salary_ranges.info()

# another method to check for missing values
salary_ranges.isnull().sum()

# Show descriptive stats:
# Three quantitative columns: 
# 1. Step 
# 2. Union Code 
# 3. Extended Step
salary_ranges.describe()

# We are more intertested in: 
# 1. a single quantitative column, the Biweekly High Rate 
# 2. a single qualitative column, Grade (the type of job)
# Pull out 'Biweekly High Rate' & 'Grade'
salary_ranges = salary_ranges[['Biweekly High Rate', 'Grade']]
salary_ranges.head()

# Show 'Biweekly High Rate' descriptive stats:
salary_ranges['Biweekly High Rate'].describe()

# Clean our Biweekly High columns to remove the dollar sign.
salary_ranges['Biweekly High Rate'] = salary_ranges['Biweekly High Rate'].map(lambda value: value.replace('

# Check to see the '

# Convert the 'Biweekly High Rate' columns to float.
salary_ranges['Biweekly High Rate'] = salary_ranges['Biweekly High Rate'].astype(float)

# Convert the 'Grade' columns to string.
salary_ranges['Grade'] = salary_ranges['Grade'].astype(str)

# Check to see if converting the data types worked
salary_ranges.info()

# 'Grade' column is Nominal level.
# Basic Value Counts of the Grade column.
# Grade 0000 has the number of 61.
# Grade 07450 has the number of 12.
# Grade 06870 has the number of 9, and so on.
salary_ranges['Grade'].value_counts().head()

# Bar Chart of the 'Grade' column
salary_ranges['Grade'].value_counts().sort_values(ascending=False).head(5).plot(kind='bar')

# the 'Grade' column as a pie chart (top 5 values only)
salary_ranges['Grade'].value_counts().sort_values(ascending=False).head(5).plot(kind='pie')

# load in the data set; how much people enjoy the San Francisco International Airport or SFO. 
customer_enjoy = pd.read_csv('./data/2013_SFO_Customer_survey.csv')

# '2013_SFO_Customer_survey.csv' has 3535 rows, 95 columns.
customer_enjoy.shape

# Feature 'Q7A_ART': artwork and exhibitions. 
# Feature 'Q7A_ART'; The possible choices are 0, 1, 2, 3, 4, 5, 6.
# 1: Unacceptable
# 2: Below Average
# 3: Average
# 4: Good
# 5: Outstanding
# 6: Have Never Used or Visited
# 0: Blank
# Even though the values of Feature 'Q7A_ART' are numbers, those numbers represent a category.
# Feature 'Q7A_ART' belongs to the qualitative side, and more specifically, ordinal.
art_ratings = customer_enjoy['Q7A_ART'] 

# Show descriptive stats:
art_ratings.describe()

# Only consider ratings 1-5
art_ratings = art_ratings[(art_ratings >= 1) & (art_ratings <= 5)]

# Cast the values as strings
art_ratings = art_ratings.astype(str) 

# Show descriptive stats:
art_ratings.describe()

# Ordinal level can use pie charts, just like in nominal level.
art_ratings.value_counts().plot(kind='pie')

#  Ordinal level can use bar charts, just like in nominal level.
art_ratings.value_counts().plot(kind='bar')

# Boxplots are available at the ordinal level.
art_ratings.value_counts().plot(kind='box')

# load in the data set.
# each row quantifies the average temperature of cities around the world by the month.
# going back to the 18th century. 
climate = pd.read_csv('./data/GlobalLandTemperaturesByCity.csv')
climate.head()

# see if we have any missing values.
# 'AverageTemperature' has 364130 missing values.
# 'AverageTemperatureUncertainty' also has 364130 missing values.
climate.isnull().sum() 

# remove missing values in order to get a better look.
climate.dropna(axis=0, inplace=True)

climate.head()

# see if we have any missing values.
# No missing values; all good.
climate.isnull().sum()

# the number of 'AverageTemperature' is 111994.
# 'AverageTemperature' has too many values. 
# we cannot use a bar/pie chart for 'AverageTemperature'.
climate['AverageTemperature'].nunique()

# the most common graph to utilize starting at interval level is "histogram". 
# "histogram"; visualizes buckets of quantities and shows frequencies of these buckets.
# a histogram for the 'AverageTemperature' around the world.
# see the distribution of 'AverageTemperature' in a very holistic view.
climate['AverageTemperature'].hist()

# Show descriptive stats.
climate['AverageTemperature'].describe()

# convert the dt column to datetime.
climate['dt'] = pd.to_datetime(climate['dt'])

# extract the year 
climate['year'] = climate['dt'].map(lambda value: value.year)

climate.head()

# subset the data to only be the temperatures recorded in the China:
climate_sub_china = climate.loc[climate['Country'] == 'China']

# year 1983 would become 20.
# year 1750 would become 18.
climate_sub_china['century'] = climate_sub_china['year'].map(lambda x: x/100+1)

climate_sub_china['century'] = climate_sub_china['century'].round().astype(int) 

climate_sub_china.head()

# with the new column century
# let's plot four histograms of temperature, one for each century.
climate_sub_china['AverageTemperature'].hist(by=climate_sub_china['century'],
    sharex=True, sharey=True,
    figsize=(10, 10),
    bins=10)

climate_sub_china.groupby('century')['AverageTemperature'].mean().plot(kind='line')

centry_changes_china = climate_sub_china.groupby('century')['AverageTemperature'].mean()

centry_changes_china

centry_changes_china[21] - centry_changes_china[19]

# oof, that's not pretty. It seems to be a lot of noise, and that is to be expected.
# every year has lots of 'AverageTemperature'.
# so we see many vertical points (the values of 'AverageTemperature') at each year.
x = climate_sub_china['year']
y = climate_sub_china['AverageTemperature']
fig, ax = plt.subplots(figsize=(10,5))
ax.scatter(x, y)
plt.show()

#  employ a groupby the year column to do "mean" ; remove much of this noise.
climate_sub_china.groupby('year').mean()['AverageTemperature'].plot()

# taking a rolling mean over the years; rolling = 10.
# rolling = 10; time windows is 10 years.
# time windows is 10 years; plot the mean of 'AverageTemperature' in every 10 years interval.
climate_sub_china.groupby('year').mean()['AverageTemperature'].rolling(10).mean().plot()

# What is the average rate across all of the Grades
# Which 20 Grades have the highest Biweekly high rate
fig = plt.figure(figsize=(15,5))
ax = fig.gca()

salary_ranges.groupby('Grade')[['Biweekly High Rate']].mean().sort_values(
    'Biweekly High Rate', ascending=False).head(20).plot.bar(stacked=False,
                                                             ax=ax, color='darkorange') 

ax.set_title('Top 20 Grade by Mean Biweekly High Rate')

# Which 20 Grades have the lowest Biweekly high rate
fig = plt.figure(figsize=(15,5))
ax = fig.gca()

salary_ranges.groupby('Grade')[['Biweekly High Rate']].mean().sort_values(
    'Biweekly High Rate', ascending=False).tail(20).plot.bar(stacked=False,
                                                             ax=ax, color='darkorange') 

ax.set_title('Bottom 20 Grade by Mean Biweekly High Rate')

# money is at the ratio level.
# we can also find the ratio of the highest-paid employee to the lowest-paid employee.
salary_df = salary_ranges.groupby('Grade')[['Biweekly High Rate']].mean().sort_values(
    'Biweekly High Rate', ascending=False)

salary_df.iloc[0][0] / salary_df.iloc[-1][0]