Python File Handling: Read, Write, and Manage Files

Reading and writing files is one of the most common tasks in Python. Whether you're processing logs, loading configs, or saving results, Python's built-in open() function and the modern pathlib module cover everything you need.

Quick Reference

Opening Files with open()

The open() function is the foundation. Always use it as a context manager with with — it closes the file automatically, even if an exception occurs.

# Read mode (default)
with open("notes.txt", "r", encoding="utf-8") as f:
    content = f.read()

# Write mode (creates or overwrites)
with open("output.txt", "w", encoding="utf-8") as f:
    f.write("Hello, world!\n")

# Append mode (adds to end, creates if missing)
with open("log.txt", "a", encoding="utf-8") as f:
    f.write("New entry\n")

# Exclusive creation (fails if file exists — prevents overwrite)
with open("config.txt", "x", encoding="utf-8") as f:
    f.write("initial config")

File Mode Reference

Always specify encoding="utf-8" for text files to avoid platform-specific surprises (cp1252 on Windows vs utf-8 on Linux).

Reading Files

# Read entire file as one string
with open("notes.txt", encoding="utf-8") as f:
    text = f.read()

# Read all lines into a list (includes \n at end of each line)
with open("notes.txt", encoding="utf-8") as f:
    lines = f.readlines()           # ["line1\n", "line2\n", ...]
    lines = [l.rstrip() for l in lines]  # strip newlines

# Read one line at a time (memory-efficient for large files)
with open("large.log", encoding="utf-8") as f:
    for line in f:                  # file object is iterable
        process(line.rstrip())

# Read a single line
with open("notes.txt", encoding="utf-8") as f:
    first = f.readline()            # "line1\n"

Cleanest way: splitlines()

lines = open("notes.txt", encoding="utf-8").read().splitlines()
# Strips newlines, handles \r\n on Windows automatically

Writing Files

# Write a string
with open("output.txt", "w", encoding="utf-8") as f:
    f.write("First line\n")
    f.write("Second line\n")

# Write multiple lines at once
lines = ["apple\n", "banana\n", "cherry\n"]
with open("fruits.txt", "w", encoding="utf-8") as f:
    f.writelines(lines)             # No separator added — include \n yourself

# Append a log entry
import datetime
with open("app.log", "a", encoding="utf-8") as f:
    ts = datetime.datetime.now().isoformat()
    f.write(f"[{ts}] Server started\n")

pathlib — The Modern Way

pathlib.Path is cleaner than os.path and handles slashes cross-platform. Available since Python 3.4.

from pathlib import Path

# Build paths (forward slash works on all platforms)
base = Path("/data/project")
config_file = base / "config" / "settings.json"

# Read and write in one line
text = Path("notes.txt").read_text(encoding="utf-8")
Path("output.txt").write_text("hello\n", encoding="utf-8")

# Read binary
data = Path("image.png").read_bytes()
Path("copy.png").write_bytes(data)

# File info
p = Path("notes.txt")
print(p.exists())       # True / False
print(p.is_file())      # True
print(p.is_dir())       # False
print(p.suffix)         # ".txt"
print(p.stem)           # "notes"
print(p.name)           # "notes.txt"
print(p.parent)         # Path(".")
print(p.stat().st_size) # file size in bytes

# List directory
for f in Path(".").iterdir():
    print(f)

# Glob pattern matching
for md in Path("docs").glob("**/*.md"):   # recursive
    print(md)

# Create directories
Path("logs/2024").mkdir(parents=True, exist_ok=True)

# Rename and delete
p.rename("notes_backup.txt")
Path("old.txt").unlink(missing_ok=True)   # delete, no error if missing

CSV Files

import csv

# Read CSV as list of dicts (header row becomes keys)
with open("users.csv", encoding="utf-8", newline="") as f:
    reader = csv.DictReader(f)
    users = list(reader)
# [{"name": "Alice", "age": "30"}, ...]

# Write CSV from list of dicts
fields = ["name", "age", "email"]
rows = [
    {"name": "Alice", "age": 30, "email": "alice@example.com"},
    {"name": "Bob",   "age": 25, "email": "bob@example.com"},
]
with open("output.csv", "w", encoding="utf-8", newline="") as f:
    writer = csv.DictWriter(f, fieldnames=fields)
    writer.writeheader()
    writer.writerows(rows)

# Read as plain lists (no header handling)
with open("data.csv", encoding="utf-8", newline="") as f:
    reader = csv.reader(f)
    next(reader)          # skip header
    for row in reader:
        name, age = row[0], row[1]

Always pass newline="" when opening CSV files — the csv module handles line endings itself.

JSON Files

import json
from pathlib import Path

# Read JSON file
data = json.loads(Path("config.json").read_text(encoding="utf-8"))

# Write JSON file (pretty-printed)
config = {"host": "localhost", "port": 8080, "debug": False}
Path("config.json").write_text(
    json.dumps(config, indent=2, ensure_ascii=False),
    encoding="utf-8"
)

# Using open() directly (streams — good for large files)
with open("large.json", encoding="utf-8") as f:
    data = json.load(f)

with open("output.json", "w", encoding="utf-8") as f:
    json.dump(data, f, indent=2)

Binary Files

# Copy an image
with open("photo.jpg", "rb") as src, open("copy.jpg", "wb") as dst:
    dst.write(src.read())

# Read in chunks (memory-efficient for large files)
CHUNK = 65_536  # 64 KB
with open("bigfile.bin", "rb") as f:
    while chunk := f.read(CHUNK):
        process(chunk)

# Seek to a position
with open("data.bin", "rb") as f:
    f.seek(100)           # jump to byte 100
    header = f.read(16)   # read 16 bytes
    pos = f.tell()        # current position

Error Handling

from pathlib import Path

def read_config(path: str) -> dict:
    try:
        text = Path(path).read_text(encoding="utf-8")
        return json.loads(text)
    except FileNotFoundError:
        raise FileNotFoundError(f"Config not found: {path}")
    except PermissionError:
        raise PermissionError(f"No read permission: {path}")
    except json.JSONDecodeError as e:
        raise ValueError(f"Invalid JSON in {path}: {e}")

# Check before opening (race condition possible — use try/except in production)
p = Path("settings.json")
if p.exists():
    data = json.loads(p.read_text(encoding="utf-8"))
else:
    data = {}

File Operations with shutil and os

import shutil
import os
from pathlib import Path

# Copy (shutil.copy2 preserves metadata)
shutil.copy2("src.txt", "dst.txt")
shutil.copy2("src.txt", "backup/")    # copy into directory

# Move / rename
shutil.move("old.txt", "archive/old.txt")

# Copy entire directory tree
shutil.copytree("src_dir", "dst_dir")

# Delete directory tree
shutil.rmtree("build/")               # careful — no recycle bin!

# Delete file
Path("temp.txt").unlink(missing_ok=True)

# File size
size = os.path.getsize("video.mp4")   # bytes
size = Path("video.mp4").stat().st_size

# List files matching pattern
import glob
logs = glob.glob("logs/*.log")
logs = list(Path("logs").glob("*.log"))  # pathlib equivalent

# Temporary files (auto-deleted)
import tempfile
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=True) as tmp:
    tmp.write("scratch data")
    tmp.flush()
    process(tmp.name)   # file exists here
# file is deleted here

Common Mistakes

FAQ

Should I use open() or pathlib.Path? Use Path.read_text() / write_text() for simple reads/writes — it's cleaner. Use open() when you need fine-grained control (seek, chunk reading, flush).

How do I read a file line by line without loading it all? for line in open("file.txt", encoding="utf-8"): — the file object is a lazy iterator, reading one line at a time.

What's the difference between read() and readlines()? read() returns one big string. readlines() returns a list of strings (with \n). For most use cases, read().splitlines() is the cleanest option.

How do I handle files that may not exist? Wrap in try/except FileNotFoundError, or check Path("f").exists() first (but prefer try/except in concurrent code).

How do I write UTF-8 with BOM for Excel compatibility? Use encoding="utf-8-sig" when writing CSV files you'll open in Excel.

What's the best way to process a multi-GB file? Read in chunks: while chunk := f.read(65536): or iterate line by line with for line in f:. Never call f.read() on huge files.