Importing data in python
This is part of my data science learning journey with Datacamp and in this blog post we will cover about different aspects of importing data in python. You can see detail execution of these files at kaggle and github
Introduction to Importing Data in Python
Introduction and flat files
# Open a file: file
file = open("moby_dick.txt", mode="r")
# Print it
print(file.read())
# Check whether file is closed
print(file.closed)
# Close file
file.close()
# Check whether file is closed
print(file.closed)
Importing text files line by line
# Read & print the first 3 lines
with open('moby_dick.txt',mode='r') as file:
print(file.readline())
print(file.readline())
print(file.readline())
Using NumPy to import flat files
# Import package
import numpy as np
# Assign filename to variable: file
file = 'digits.csv'
# Load file as array: digits
digits = np.loadtxt(file, delimiter=',')
# Print datatype of digits
print(type(digits))
# Select and reshape a row
im = digits[21, 1:]
im_sq = np.reshape(im, (28, 28))
# Plot reshaped data (matplotlib.pyplot already loaded as plt)
plt.imshow(im_sq, cmap='Greys', interpolation='nearest')
plt.show()
Customizing your NumPy import
# Import numpy
import numpy as np
# Assign the filename: file
file = 'digits_header.txt'
# Load the data: data
data = np.loadtxt(file, delimiter='\t', skiprows=1, usecols=[0,2])
# Print data
print(data)
Importing different data types
# Assign filename: file
file = 'seaslug.txt'
# Import file: data
data = np.loadtxt(file, delimiter='\t', dtype=str)
# Print the first element of data
print(data[0])
# Import data as floats and skip the first row: data_float
data_float = np.loadtxt(file, delimiter='\t', dtype=float, skiprows=1)
# Print the 10th element of data_float
print(data[9])
# Plot a scatterplot of the data
plt.scatter(data_float[:, 0], data_float[:, 1])
plt.xlabel('time (min.)')
plt.ylabel('percentage of larvae')
plt.show()
# Assign the filename: file
file = 'titanic.csv'
# Import file using np.recfromcsv: d
d=np.recfromcsv(file,delimiter=',',names=True,dtype=None)
# Print out first three entries of d
print(d[:3])
Customizing your pandas import
# Import matplotlib.pyplot as plt
import matplotlib.pyplot as plt
# Assign filename: file
file = 'titanic_corrupt.txt'
# Import file: data
data = pd.read_csv(file, sep='\t', comment='#', na_values='Nothin')
# Print the head of the DataFrame
print(data.head())
# Plot 'Age' variable in a histogram
pd.DataFrame.hist(data[['Age']])
plt.xlabel('Age (years)')
plt.ylabel('count')
plt.show()
Importing data from other file types
Loading a pickled file
# Import pickle package
import pickle
# Open pickle file and load data: d
with open('data.pkl', 'rb') as file:
d = pickle.load(file)
# Print d
print(d)
# Print datatype of d
print(type(d))
Listing sheets in Excel files
# Import pandas
import pandas as pd
# Assign spreadsheet filename: file
file = 'battledeath.xlsx'
# Load spreadsheet: xls
xls = pd.ExcelFile(file)
# Print sheet names
print(xls.sheet_names)
Importing sheets from Excel files
# Load a sheet into a DataFrame by name: df1
df1 = xls.parse('2004')
# Print the head of the DataFrame df1
print(df1.head())
# Load a sheet into a DataFrame by index: df2
df2 = xls.parse('2002')
# Print the head of the DataFrame df2
print(df2.head())
Customizing your spreadsheet import
# Import sas7bdat package
from sas7bdat import SAS7BDAT
# Save file to a DataFrame: df_sas
with SAS7BDAT('sales.sas7bdat') as file:
df_sas=file.to_data_frame()
# Print head of DataFrame
print(df_sas.head())
# Plot histogram of DataFrame features (pandas and pyplot already imported)
pd.DataFrame.hist(df_sas[['P']])
plt.ylabel('count')
plt.show()
Using read_stata to import Stata files
# Import pandas
import pandas as pd
# Load Stata file into a pandas DataFrame: df
df=pd.read_stata('disarea.dta')
# Print the head of the DataFrame df
print(df.head())
# Plot histogram of one column of the DataFrame
pd.DataFrame.hist(df[['disa10']])
plt.xlabel('Extent of disease')
plt.ylabel('Number of countries')
plt.show()
Using h5py to import HDF5 files
# Import packages
import numpy as np
import h5py
# Assign filename: file
file='LIGO_data.hdf5'
# Load file: data
data = h5py.File(file, 'r')
# Print the datatype of the loaded file
print(type(data))
# Print the keys of the file
for key in data.keys():
print(key)
Extracting data from your HDF5 file
# Get the HDF5 group: group
group=data['strain']
# Check out keys of group
for key in group.keys():
print(key)
# Set variable equal to time series data: strain
strain=data['strain']['Strain'].value
# Set number of time points to sample: num_samples
num_samples=10000
# Set time vector
time = np.arange(0, 1, 1/num_samples)
# Plot data
plt.plot(time, strain[:num_samples])
plt.xlabel('GPS Time (s)')
plt.ylabel('strain')
plt.show()
Importing & exploring structure of matlab file
# Print the keys of the MATLAB dictionary
print(mat.keys())
# Print the type of the value corresponding to the key 'CYratioCyt'
print(type(mat['CYratioCyt']))
# Print the shape of the value corresponding to the key 'CYratioCyt'
print(np.shape(mat['CYratioCyt']))
# Subset the array and plot it
data = mat['CYratioCyt'][25, 5:]
fig = plt.figure()
plt.plot(data)
plt.xlabel('time (min.)')
plt.ylabel('normalized fluorescence (measure of expression)')
plt.show()
Working with relational databases in Python
# Import necessary module
from sqlalchemy import create_engine
# Create engine: engine
engine=create_engine('sqlite:///Chinook.sqlite')
# Save the table names to a list: table_names
table_names=engine.table_names()
# Print the table names to the shell
print(table_names)
# Import packages
from sqlalchemy import create_engine
import pandas as pd
# Create engine: engine
engine = create_engine('sqlite:///Chinook.sqlite')
# Open engine connection: con
con=engine.connect()
# Perform query: rs
rs = con.execute("select * from album")
# Save results of the query to DataFrame: df
df = pd.DataFrame(rs.fetchall())
# Close connection
con.close()
# Print head of DataFrame df
print(df.head())
# Open engine in context manager
# Perform query and save results to DataFrame: df
with engine.connect() as con:
rs = con.execute("select lastname, title from employee")
df = pd.DataFrame(rs.fetchmany(size=3))
df.columns = rs.keys()
# Print the length of the DataFrame df
print(len(df))
# Print the head of the DataFrame df
print(df.head())
# Create engine: engine
engine = create_engine('sqlite:///Chinook.sqlite')
# Open engine in context manager
# Perform query and save results to DataFrame: df
with engine.connect() as con:
rs = con.execute("select * from employee where employeeid>=6")
df = pd.DataFrame(rs.fetchall())
df.columns = rs.keys()
# Print the head of the DataFrame df
print(df.head())
# Create engine: engine
engine = create_engine('sqlite:///Chinook.sqlite')
# Open engine in context manager
with engine.connect() as con:
rs = con.execute('select * from employee order by birthdate')
df = pd.DataFrame(rs.fetchall())
# Set the DataFrame's column names
df.columns=rs.keys()
# Print head of DataFrame
print(df.head())
Using panda to query sql
# Import packages
from sqlalchemy import create_engine
import pandas as pd
# Create engine: engine
engine = create_engine('sqlite:///Chinook.sqlite')
# Execute query and store records in DataFrame: df
df = pd.read_sql_query("select * from album", engine)
# Print head of DataFrame
print(df.head())
# Open engine in context manager and store query result in df1
with engine.connect() as con:
rs = con.execute("SELECT * FROM Album")
df1 = pd.DataFrame(rs.fetchall())
df1.columns = rs.keys()
# Confirm that both methods yield the same result
print(df.equals(df1))
# Import packages
from sqlalchemy import create_engine
import pandas as pd
# Create engine: engine
engine=create_engine('sqlite:///Chinook.sqlite')
# Execute query and store records in DataFrame: df
df=pd.read_sql_query("select * from Employee where employeeid >=6 order by birthdate",engine)
# Print head of DataFrame
print(df.head())
With Joins
# Open engine in context manager
# Perform query and save results to DataFrame: df
with engine.connect() as con:
rs=con.execute("select title,name from album inner join artist on album.artistid=artist.artistid")
df=pd.DataFrame(rs.fetchall())
df.columns=rs.keys()
# Print head of DataFrame df
print(df.head())
# Execute query and store records in DataFrame: df
df=pd.read_sql_query("select * from playlisttrack inner join track on playlisttrack.trackid=track.trackid where milliseconds < 250000",engine)
# Print head of DataFrame
print(df.head())
Importing data from the Internet
Importing flat files from the web
# Import package
from urllib.request import urlretrieve
# Import pandas
import pandas as pd
# Assign url of file: url
url='https://s3.amazonaws.com/assets.datacamp.com/production/course_1606/datasets/winequality-red.csv'
# Save file locally
urlretrieve(url,'winequality-red.csv')
# Read file into a DataFrame and print its head
df = pd.read_csv('winequality-red.csv', sep=';')
print(df.head())
# Import packages
import matplotlib.pyplot as plt
import pandas as pd
# Assign url of file: url
url='https://s3.amazonaws.com/assets.datacamp.com/production/course_1606/datasets/winequality-red.csv'
# Read file into a DataFrame: df
df=pd.read_csv(url,sep=';')
# Print the head of the DataFrame
print(df.head())
# Plot first column of df
pd.DataFrame.hist(df.ix[:, 0:1])
plt.xlabel('fixed acidity (g(tartaric acid)/dm$^3$)')
plt.ylabel('count')
plt.show()
Importing non-flat files from the web
# Import package
import pandas as pd
# Assign url of file: url
url='http://s3.amazonaws.com/assets.datacamp.com/course/importing_data_into_r/latitude.xls'
# Read in all sheets of Excel file: xls
xls=pd.read_excel(url,sheetname=None)
# Print the sheetnames to the shell
print(xls.keys())
# Print the head of the first sheet (using its name, NOT its index)
print(xls['1700'].head())
Performing HTTP requests in Python using urllib
# Import packages
from urllib.request import urlopen,Request
# Specify the url
url = "https://campus.datacamp.com/courses/1606/4135?ex=2"
# This packages the request: request
request=Request(url)
# Sends the request and catches the response: response
response=urlopen(request)
# Print the datatype of response
print(type(response))
# Be polite and close the response!
response.close()
Printing HTTP request results in Python using urllib
# Import packages
from urllib.request import urlopen, Request
# Specify the url
url = "https://campus.datacamp.com/courses/1606/4135?ex=2"
# This packages the request
request = Request(url)
# Sends the request and catches the response: response
response=urlopen(request)
# Extract the response: html
html=response.read()
# Print the html
print(html)
# Be polite and close the response!
response.close()
Performing HTTP requests in Python using requests
# Import package
import requests
# Specify the url: url
url="http://www.datacamp.com/teach/documentation"
# Packages the request, send the request and catch the response: r
r=requests.get(url)
# Extract the response: text
text=r.text
# Print the html
print(text)
Parsing HTML with BeautifulSoup
# Import packages
import requests
from bs4 import BeautifulSoup
# Specify url: url
url='https://www.python.org/~guido/'
# Package the request, send the request and catch the response: r
r=requests.get(url)
# Extracts the response as html: html_doc
html_doc=r.text
# Create a BeautifulSoup object from the HTML: soup
soup=BeautifulSoup(html_doc)
# Prettify the BeautifulSoup object: pretty_soup
pretty_soup=soup.prettify()
# Print the response
print(pretty_soup)
Turning a webpage into data using BeautifulSoup: getting the text
# Import packages
import requests
from bs4 import BeautifulSoup
# Specify url: url
url = 'https://www.python.org/~guido/'
# Package the request, send the request and catch the response: r
r = requests.get(url)
# Extract the response as html: html_doc
html_doc = r.text
# Create a BeautifulSoup object from the HTML: soup
soup=BeautifulSoup(html_doc)
# Get the title of Guido's webpage: guido_title
guido_title=soup.title
# Print the title of Guido's webpage to the shell
print(guido_title)
# Get Guido's text: guido_text
guido_text=soup.get_text()
# Print Guido's text to the shell
print(guido_text)
Turning a webpage into data using BeautifulSoup: getting the hyperlinks
# Import packages
import requests
from bs4 import BeautifulSoup
# Specify url
url = 'https://www.python.org/~guido/'
# Package the request, send the request and catch the response: r
r = requests.get(url)
# Extracts the response as html: html_doc
html_doc = r.text
# create a BeautifulSoup object from the HTML: soup
soup = BeautifulSoup(html_doc)
# Print the title of Guido's webpage
print(soup.title)
# Find all 'a' tags (which define hyperlinks): a_tags
a_tags=soup.find_all('a')
# Print the URLs to the shell
for link in a_tags:
print(link.get('href'))
Interacting with APIs to import data from the web
Loading and exploring a JSON
# Load JSON: json_data
with open("a_movie.json") as json_file:
json_data=json.load(json_file)
# Print each key-value pair in json_data
for k in json_data.keys():
print(k + ': ', json_data[k])
API requests
# Import requests package
import requests
# Assign URL to variable: url
url='http://www.omdbapi.com/?apikey=72bc447a&t=social+network'
# Package the request, send the request and catch the response: r
r = requests.get(url)
# Print the text of the response
print(r.text)
JSON–from the web to Python
# Import package
import requests
# Assign URL to variable: url
url = 'http://www.omdbapi.com/?apikey=72bc447a&t=social+network'
# Package the request, send the request and catch the response: r
r=requests.get(url)
# Decode the JSON data into a dictionary: json_data
json_data=r.json()
# Print each key-value pair in json_data
for k in json_data.keys():
print(k + ': ', json_data[k])
Checking out the Wikipedia API
# Import package
import requests
# Assign URL to variable: url
url='https://en.wikipedia.org/w/api.php?action=query&prop=extracts&format=json&exintro=&titles=pizza'
# Package the request, send the request and catch the response: r
r = requests.get(url)
# Decode the JSON data into a dictionary: json_data
json_data=r.json()
# Print the Wikipedia page extract
pizza_extract = json_data['query']['pages']['24768']['extract']
print(pizza_extract)
Diving deep into the Twitter API
# Import package
import tweepy
# Store OAuth authentication credentials in relevant variables
access_token = "1092294848-aHN7DcRP9B4VMTQIhwqOYiB14YkW92fFO8k8EPy"
access_token_secret = "X4dHmhPfaksHcQ7SCbmZa2oYBBVSD2g8uIHXsp5CTaksx"
consumer_key = "nZ6EA0FxZ293SxGNg8g8aP0HM"
consumer_secret = "fJGEodwe3KiKUnsYJC3VRndj7jevVvXbK2D5EiJ2nehafRgA6i"
# Pass OAuth details to tweepy's OAuth handler
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token,access_token_secret)
Streaming tweets
Load and explore your Twitter data
# Import package
import json
# String of path to file: tweets_data_path
tweets_data_path='tweets.txt'
# Initialize empty list to store tweets: tweets_data
tweets_data=[]
# Open connection to file
tweets_file = open(tweets_data_path, "r")
# Read in tweets and store in list: tweets_data
for line in tweets_file:
tweet=json.loads(line)
tweets_data.append(tweet)
# Close connection to file
tweets_file.close()
# Print the keys of the first tweet dict
print(tweets_data[0].keys())
Twitter data to DataFrame
# Import package
import pandas as pd
# Build DataFrame of tweet texts and languages
df = pd.DataFrame(tweets_data, columns=['text', 'lang'])
# Print head of DataFrame
print(df.head())
basic Twitter text analysis
Plotting your Twitter data
# Import packages
import seaborn as sns
import matplotlib.pyplot as plt
# Set seaborn style
sns.set(color_codes=True)
# Create a list of labels:cd
cd = ['clinton', 'trump', 'sanders', 'cruz']
# Plot the bar chart
ax = sns.barplot(cd, [clinton, trump, sanders, cruz])
ax.set(ylabel="count")
plt.show()
0 comments:
Post a Comment