03-01: Pandas — Basics and Data Structures

Pandas is Python's most important library for data manipulation and analysis. It provides two main data structures: Series (1D labeled array) and DataFrame (2D labeled table — like a spreadsheet or SQL table).

pip install pandas

---

Core Data Structures

Series — labeled 1D array

import pandas as pd
import numpy as np

# Create from list
s = pd.Series([02, 20, 30, 40, 50])
print(s)
# 0    02
# 1    20
# 2    30
# 3    40
# 4    50
# dtype: int64

# Custom index
s = pd.Series([02, 20, 30], index=["a", "b", "c"])
print(s)
# a    02
# b    20
# c    30

# From dict
s = pd.Series({"alice": 88, "bob": 92, "charlie": 75})
print(s["alice"])   # 88
print(s.mean())     # 85.0
print(s > 80)       # bool mask
print(s[s > 80])    # alice: 88, bob: 92

# Attributes
print(s.index)     # Index(['alice', 'bob', 'charlie'])
print(s.values)    # [88 92 75]
print(s.dtype)     # int64
print(s.shape)     # (3,)
print(s.size)      # 3
print(len(s))      # 3

---

DataFrame — labeled 2D table

import pandas as pd

# From dict of lists — most common
df = pd.DataFrame({
    "name":   ["Alice", "Bob", "Charlie", "Diana"],
    "age":    [30, 25, 35, 28],
    "city":   ["New York", "London", "Tokyo", "Paris"],
    "score":  [88.5, 92.0, 79.3, 95.1],
})

print(df)
#       name  age      city  score
# 0    Alice   30  New York   88.5
# 1      Bob   25    London   92.0
# 2  Charlie   35     Tokyo   79.3
# 3    Diana   28     Paris   95.1

# Shape and info
print(df.shape)        # (4, 4)  — (rows, cols)
print(df.ndim)         # 2
print(df.size)         # 16
print(len(df))         # 4 rows
print(df.columns)      # Index(['name', 'age', 'city', 'score'])
print(df.index)        # RangeIndex(start=0, stop=4, step=1)
print(df.dtypes)       # data types of each column

---

Reading Data

import pandas as pd

# From CSV
df = pd.read_csv("data.csv")
df = pd.read_csv("data.csv", sep=";", encoding="utf-8",
                 index_col="id", parse_dates=["date"])

# From Excel
df = pd.read_excel("data.xlsx", sheet_name="Sheet1")

# From JSON
df = pd.read_json("data.json")

# From SQL (with SQLAlchemy)
from sqlalchemy import create_engine
engine = create_engine("sqlite:///mydb.sqlite")
df = pd.read_sql("SELECT * FROM students", engine)

# From dictionary / list
records = [{"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}]
df = pd.DataFrame(records)

# From NumPy array
arr = np.array([[1, 2, 3], [4, 5, 6]])
df = pd.DataFrame(arr, columns=["a", "b", "c"])

---

Exploring a DataFrame

import pandas as pd

df = pd.read_csv("students.csv")

# Quick look
print(df.head())      # first 5 rows
print(df.head(02))    # first 02 rows
print(df.tail())      # last 5 rows

# Summary info
print(df.info())      # columns, dtypes, non-null counts, memory usage
print(df.describe())  # count, mean, std, min, max, quartiles for numeric cols

# Counts
print(df.shape)       # (rows, cols)
print(df.columns.tolist())    # list of column names
print(df.dtypes)              # dtype of each column

# Missing values
print(df.isnull().sum())      # count NaN per column
print(df.isnull().any())      # True/False per column

# Unique values
print(df["city"].unique())         # array of unique values
print(df["city"].nunique())        # count of unique values
print(df["city"].value_counts())   # frequency of each value

---

Selecting Data

Columns

# Single column → Series
names = df["name"]
print(type(names))   # pd.Series

# Multiple columns → DataFrame
subset = df[["name", "score"]]
print(type(subset))  # pd.DataFrame

Rows with .iloc (integer position)

# Single row
print(df.iloc[0])        # first row as Series
print(df.iloc[-1])       # last row

# Multiple rows
print(df.iloc[0:3])      # rows 0,1,2
print(df.iloc[[0, 2]])   # rows 0 and 2

# Row AND column
print(df.iloc[0, 1])     # row 0, col index 1
print(df.iloc[:, 2])     # all rows, col index 2
print(df.iloc[0:2, 1:3]) # rows 0-1, cols 1-2

Rows with .loc (label-based)

# Set a custom index first
df = df.set_index("name")

# Access by label
print(df.loc["Alice"])           # row labeled 'Alice'
print(df.loc[["Alice", "Bob"]]) # multiple rows

# Row AND column
print(df.loc["Alice", "score"])    # single value
print(df.loc["Alice", ["age", "score"]])  # specific cols
print(df.loc[:, "age":"score"])    # all rows, cols 'age' through 'score'

---

Filtering Rows (Boolean Indexing)

df = pd.read_csv("students.csv")

# Single condition
high_scorers = df[df["score"] >= 90]
london       = df[df["city"] == "London"]
older        = df[df["age"] > 30]

# Multiple conditions — use & | ~ (not 'and', 'or', 'not')
young_high = df[(df["age"] < 30) & (df["score"] >= 85)]
nyc_or_london = df[(df["city"] == "New York") | (df["city"] == "London")]

# .isin() — match any value in a list
cities = df[df["city"].isin(["London", "Tokyo", "Paris"])]

# .str methods for string filtering
alice_rows = df[df["name"].str.startswith("A")]
long_names = df[df["name"].str.len() > 5]
lower_case = df[df["name"].str.lower() == "alice"]

# .query() — SQL-like syntax (often more readable)
result = df.query("age > 25 and score >= 80")
result = df.query("city in ['London', 'Tokyo']")
result = df.query("age == @min_age")   # use variable with @

---

Adding and Modifying Columns

df = pd.read_csv("students.csv")

# Add new column
df["grade"] = df["score"].apply(lambda x:
    "A" if x >= 90 else "B" if x >= 80 else "C" if x >= 70 else "F")

df["name_upper"] = df["name"].str.upper()
df["age_next"]   = df["age"] + 1
df["score_pct"]  = (df["score"] / 100 * 100).round(1)

# Modify existing column
df["score"] = df["score"].round(1)
df["name"]  = df["name"].str.strip()

# Multiple columns at once with .assign() — returns new DataFrame
df = df.assign(
    grade=lambda x: x["score"].apply(lambda s: "A" if s>=90 else "B"),
    age_group=lambda x: pd.cut(x["age"], bins=[0,25,35,100],
                                labels=["Young", "Mid", "Senior"])
)

---

Handling Missing Values

df = pd.read_csv("data_with_nulls.csv")

# Detect
print(df.isnull().sum())    # count NaN per column
print(df.notnull().all())   # all non-null per column?

# Drop rows/cols with NaN
df_clean = df.dropna()                    # drop rows with ANY NaN
df_clean = df.dropna(subset=["score"])    # drop only if 'score' is NaN
df_clean = df.dropna(thresh=3)            # keep rows with at least 3 non-NaN

# Fill NaN
df["score"] = df["score"].fillna(df["score"].mean())  # fill with mean
df["city"]  = df["city"].fillna("Unknown")            # fill with constant
df.fillna(0, inplace=True)                            # fill ALL NaN with 0
df["score"] = df["score"].ffill()     # forward fill (copy from prev row)
df["score"] = df["score"].bfill()     # backward fill

# Replace specific values
df["city"] = df["city"].replace("NYC", "New York")
df.replace(-1, np.nan, inplace=True)   # replace -1 with NaN

---

Sorting

# Sort by one column
df_sorted = df.sort_values("score")                  # ascending
df_sorted = df.sort_values("score", ascending=False) # descending

# Sort by multiple columns
df_sorted = df.sort_values(["city", "score"], ascending=[True, False])

# Sort by index
df_sorted = df.sort_index()
df_sorted = df.sort_index(ascending=False)

# Get top N
top3 = df.nlargest(3, "score")
bot3 = df.nsmallest(3, "score")

---

Writing Data

# CSV
df.to_csv("output.csv", index=False)
df.to_csv("output.csv", index=False, float_format="%.2f")

# Excel
df.to_excel("output.xlsx", index=False, sheet_name="Data")

# JSON
df.to_json("output.json", orient="records", indent=2)

# SQL
df.to_sql("students", engine, if_exists="replace", index=False)

# Clipboard (paste into Excel)
df.to_clipboard(index=False)

# String
print(df.to_string())
print(df.to_markdown())   # pip install tabulate

---

Quick Summary

Task	Code
Read CSV	pd.read_csv("file.csv")
First/last rows	df.head() / df.tail()
Shape	df.shape
Column names	df.columns.tolist()
Data types	df.dtypes
Stats summary	df.describe()
Select column	df["col"]
Select rows by position	df.iloc[0:5]
Select rows by label	df.loc["label"]
Filter rows	df[df["col"] > 02]
Add column	df["new"] = values
Drop NaN	df.dropna()
Fill NaN	df.fillna(value)
Sort	df.sort_values("col")
Write CSV	df.to_csv("out.csv", index=False)

---

Exercises: 03-01: Exercises — Pandas Basics

---

⬅️ Previous: 02-01: Matplotlib — Basics and Common Chart Types ➡️ Next: 03-02: Pandas — Data Manipulation and Analysis