Chapter 3: Linear Dependence and Independence

Precise Definition of Linear Dependence

Mathematical Definition

Given a set of vectors $\{\mathbf{v_1}, \mathbf{v_2}, \ldots, \mathbf{v_n}\}$, if there exist not all zero scalars $c_1, c_2, \ldots, c_n$ such that:

$c_1\mathbf{v_1} + c_2\mathbf{v_2} + \cdots + c_n\mathbf{v_n} = \mathbf{0}$

then this set of vectors is called linearly dependent.

If the above equation holds only when $c_1 = c_2 = \cdots = c_n = 0$, then this set of vectors is called linearly independent.

import numpy as np
import matplotlib.pyplot as plt
from scipy.linalg import null_space

# Example of linear dependence
def demonstrate_linear_dependence():
    print("Linear dependence examples:")
    print("=" * 50)

    # Example 1: Obviously linearly dependent vector set
    v1 = np.array([1, 2])
    v2 = np.array([2, 4])  # v2 = 2*v1
    v3 = np.array([3, 6])  # v3 = 3*v1

    print(f"Vector set: v1={v1}, v2={v2}, v3={v3}")
    print(f"Observation: v2 = 2*v1, v3 = 3*v1")
    print(f"Linear combination: 1*v1 - 0.5*v2 + 0*v3 = {1*v1 - 0.5*v2 + 0*v3}")

    # Example 2: Linearly independent vector set
    u1 = np.array([1, 0])
    u2 = np.array([0, 1])

    print(f"\nVector set: u1={u1}, u2={u2}")
    print("These two vectors are linearly independent because no coefficients other than zero make the linear combination zero")

demonstrate_linear_dependence()

Geometric Intuition

Linear Dependence in 2D Space

def visualize_2d_linear_dependence():
    fig, axes = plt.subplots(1, 3, figsize=(15, 5))

    # Case 1: Linearly independent (two vectors not collinear)
    v1 = np.array([3, 1])
    v2 = np.array([1, 2])

    axes[0].arrow(0, 0, v1[0], v1[1], head_width=0.2, head_length=0.2,
                  fc='blue', ec='blue', label='v1')
    axes[0].arrow(0, 0, v2[0], v2[1], head_width=0.2, head_length=0.2,
                  fc='red', ec='red', label='v2')
    axes[0].set_xlim(-1, 4)
    axes[0].set_ylim(-1, 3)
    axes[0].grid(True, alpha=0.3)
    axes[0].legend()
    axes[0].set_title('Linearly Independent\n(Not collinear)')
    axes[0].set_aspect('equal')

    # Case 2: Linearly dependent (two vectors collinear)
    w1 = np.array([2, 1])
    w2 = np.array([4, 2])  # w2 = 2*w1

    axes[1].arrow(0, 0, w1[0], w1[1], head_width=0.2, head_length=0.2,
                  fc='blue', ec='blue', label='w1')
    axes[1].arrow(0, 0, w2[0], w2[1], head_width=0.2, head_length=0.2,
                  fc='red', ec='red', label='w2=2w1')
    axes[1].set_xlim(-1, 5)
    axes[1].set_ylim(-1, 3)
    axes[1].grid(True, alpha=0.3)
    axes[1].legend()
    axes[1].set_title('Linearly Dependent\n(Collinear)')
    axes[1].set_aspect('equal')

    # Case 3: Linear dependence of three vectors
    x1 = np.array([2, 1])
    x2 = np.array([1, 2])
    x3 = np.array([3, 3])  # x3 = x1 + x2

    axes[2].arrow(0, 0, x1[0], x1[1], head_width=0.2, head_length=0.2,
                  fc='blue', ec='blue', label='x1')
    axes[2].arrow(0, 0, x2[0], x2[1], head_width=0.2, head_length=0.2,
                  fc='red', ec='red', label='x2')
    axes[2].arrow(0, 0, x3[0], x3[1], head_width=0.2, head_length=0.2,
                  fc='green', ec='green', label='x3=x1+x2')

    # Show that x3 is a linear combination of x1 and x2
    axes[2].arrow(x1[0], x1[1], x2[0], x2[1], head_width=0.1, head_length=0.1,
                  fc='orange', ec='orange', linestyle='--', alpha=0.7)

    axes[2].set_xlim(-1, 4)
    axes[2].set_ylim(-1, 4)
    axes[2].grid(True, alpha=0.3)
    axes[2].legend()
    axes[2].set_title('Linearly Dependent\n(Three vectors)')
    axes[2].set_aspect('equal')

    plt.tight_layout()
    plt.show()

visualize_2d_linear_dependence()

Linear Dependence in 3D Space

def visualize_3d_linear_dependence():
    fig = plt.figure(figsize=(15, 5))

    # 3D linearly independent vector set
    ax1 = fig.add_subplot(131, projection='3d')
    v1 = np.array([1, 0, 0])
    v2 = np.array([0, 1, 0])
    v3 = np.array([0, 0, 1])

    ax1.quiver(0, 0, 0, v1[0], v1[1], v1[2], color='red', label='e1')
    ax1.quiver(0, 0, 0, v2[0], v2[1], v2[2], color='green', label='e2')
    ax1.quiver(0, 0, 0, v3[0], v3[1], v3[2], color='blue', label='e3')
    ax1.set_xlim([0, 1.2])
    ax1.set_ylim([0, 1.2])
    ax1.set_zlim([0, 1.2])
    ax1.set_title('Linearly Independent\n(Standard basis)')
    ax1.legend()

    # 3D linearly dependent vector set (coplanar)
    ax2 = fig.add_subplot(132, projection='3d')
    u1 = np.array([1, 0, 0])
    u2 = np.array([0, 1, 0])
    u3 = np.array([1, 1, 0])  # u3 = u1 + u2

    ax2.quiver(0, 0, 0, u1[0], u1[1], u1[2], color='red', label='u1')
    ax2.quiver(0, 0, 0, u2[0], u2[1], u2[2], color='green', label='u2')
    ax2.quiver(0, 0, 0, u3[0], u3[1], u3[2], color='blue', label='u3=u1+u2')
    ax2.set_xlim([0, 1.2])
    ax2.set_ylim([0, 1.2])
    ax2.set_zlim([0, 1.2])
    ax2.set_title('Linearly Dependent\n(Coplanar)')
    ax2.legend()

    # 3D linearly dependent vector set (collinear)
    ax3 = fig.add_subplot(133, projection='3d')
    w1 = np.array([1, 1, 1])
    w2 = np.array([2, 2, 2])  # w2 = 2*w1
    w3 = np.array([0.5, 0.5, 0.5])  # w3 = 0.5*w1

    ax3.quiver(0, 0, 0, w1[0], w1[1], w1[2], color='red', label='w1')
    ax3.quiver(0, 0, 0, w2[0], w2[1], w2[2], color='green', label='w2=2w1')
    ax3.quiver(0, 0, 0, w3[0], w3[1], w3[2], color='blue', label='w3=0.5w1')
    ax3.set_xlim([0, 2.5])
    ax3.set_ylim([0, 2.5])
    ax3.set_zlim([0, 2.5])
    ax3.set_title('Linearly Dependent\n(Collinear)')
    ax3.legend()

    plt.tight_layout()
    plt.show()

visualize_3d_linear_dependence()

Methods for Determining Linear Dependence

Method 1: Homogeneous Linear System

Determining the linear dependence of a vector set $\{\mathbf{v_1}, \mathbf{v_2}, \ldots, \mathbf{v_n}\}$ is equivalent to solving the homogeneous linear system:

$\begin{bmatrix} | & | & \cdots & | \\ \mathbf{v_1} & \mathbf{v_2} & \cdots & \mathbf{v_n} \\ | & | & \cdots & | \end{bmatrix} \begin{bmatrix} c_1 \\ c_2 \\ \vdots \\ c_n \end{bmatrix} = \begin{bmatrix} 0 \\ 0 \\ \vdots \\ 0 \end{bmatrix}$

def check_linear_dependence_matrix_method(vectors):
    """
    Check linear dependence using matrix method
    """
    # Form matrix with vectors as columns
    A = np.column_stack(vectors)
    print(f"Vector matrix A:")
    print(A)

    # Calculate matrix rank
    rank = np.linalg.matrix_rank(A)
    n_vectors = len(vectors)

    print(f"Rank of matrix: {rank}")
    print(f"Number of vectors: {n_vectors}")

    if rank < n_vectors:
        print("Vectors are linearly dependent")

        # Find linear dependence relation
        # Calculate null space
        null_sp = null_space(A)
        if null_sp.size > 0:
            coeffs = null_sp[:, 0]  # Take first null space vector
            print(f"Coefficients of linear dependence: {coeffs}")

            # Verify linear combination
            linear_combination = sum(c * v for c, v in zip(coeffs, vectors))
            print(f"Verification: {' + '.join([f'{c:.3f}*v{i+1}' for i, c in enumerate(coeffs)])} = {linear_combination}")

    else:
        print("Vectors are linearly independent")

    return rank < n_vectors

# Test examples
print("Example 1: Linearly dependent vector set")
vectors1 = [np.array([1, 2, 3]), np.array([2, 4, 6]), np.array([1, 1, 1])]
check_linear_dependence_matrix_method(vectors1)

print("\n" + "="*60 + "\n")

print("Example 2: Linearly independent vector set")
vectors2 = [np.array([1, 0, 0]), np.array([0, 1, 0]), np.array([0, 0, 1])]
check_linear_dependence_matrix_method(vectors2)

Method 2: Determinant Method (Square Matrix Case)

For $n$ vectors in $n$ dimensions, we can use the determinant to determine:

def check_dependence_determinant(vectors):
    """
    Check linear dependence using determinant method for square matrices
    """
    A = np.column_stack(vectors)

    if A.shape[0] != A.shape[1]:
        print("Not a square matrix, cannot use determinant method")
        return None

    det = np.linalg.det(A)
    print(f"Matrix A:")
    print(A)
    print(f"Determinant value: {det:.6f}")

    if abs(det) < 1e-10:  # Consider numerical error
        print("Determinant is 0, vectors are linearly dependent")
        return True
    else:
        print("Determinant is non-zero, vectors are linearly independent")
        return False

# Test examples
print("Using determinant method:")
vectors_square = [np.array([1, 2]), np.array([3, 4])]
check_dependence_determinant(vectors_square)

print("\n")
vectors_dependent = [np.array([1, 2]), np.array([2, 4])]
check_dependence_determinant(vectors_dependent)

Important Properties of Linear Dependence

Property Theorems

def demonstrate_linear_dependence_properties():
    print("Important properties of linear dependence:")
    print("=" * 50)

    # Property 1: Vector set containing zero vector is linearly dependent
    print("Property 1: Vector set containing zero vector is linearly dependent")
    v1 = np.array([1, 2])
    v2 = np.array([0, 0])  # Zero vector
    v3 = np.array([3, 4])

    print(f"Vector set: {v1}, {v2}, {v3}")
    print(f"Because it contains zero vector, it is linearly dependent")
    print(f"For example: 0*{v1} + 1*{v2} + 0*{v3} = {0*v1 + 1*v2 + 0*v3}")

    print("\n" + "-"*40)

    # Property 2: Two vectors are linearly dependent if and only if one is a multiple of the other
    print("Property 2: Two vectors are linearly dependent ⟺ one is a multiple of the other")
    u1 = np.array([3, 6])
    u2 = np.array([1, 2])  # u1 = 3*u2

    print(f"Vector u1: {u1}")
    print(f"Vector u2: {u2}")
    print(f"u1 = 3*u2: {np.array_equal(u1, 3*u2)}")

    print("\n" + "-"*40)

    # Property 3: If a vector set is linearly independent, any subset is also linearly independent
    print("Property 3: Subset of linearly independent set is still linearly independent")
    basis = [np.array([1, 0, 0]), np.array([0, 1, 0]), np.array([0, 0, 1])]
    subset = [np.array([1, 0, 0]), np.array([0, 1, 0])]

    print("3D standard basis is linearly independent")
    print("Any subset of two vectors is also linearly independent")

    print("\n" + "-"*40)

    # Property 4: If a vector set is linearly dependent, at least one vector can be expressed as a linear combination of others
    print("Property 4: Linearly dependent ⟺ at least one vector can be expressed as linear combination of others")
    w1 = np.array([1, 2])
    w2 = np.array([2, 1])
    w3 = np.array([3, 3])  # w3 = w1 + w2

    print(f"Vector set: {w1}, {w2}, {w3}")
    print(f"w3 = w1 + w2: {np.array_equal(w3, w1 + w2)}")
    print("Therefore the vector set is linearly dependent")

demonstrate_linear_dependence_properties()

Maximal Linearly Independent Set

Definition and Finding Method

def find_maximal_linearly_independent_set(vectors):
    """
    Find the maximal linearly independent set of a vector set
    """
    vectors = [np.array(v) for v in vectors]
    n = len(vectors)

    print(f"Original vector set: {vectors}")

    # Incrementally add vectors, checking linear independence
    independent_set = []
    independent_indices = []

    for i, v in enumerate(vectors):
        # Add current vector to test set
        test_set = independent_set + [v]

        # Check if still linearly independent
        if len(test_set) == 1:
            # Single non-zero vector is always linearly independent
            if not np.allclose(v, 0):
                independent_set.append(v)
                independent_indices.append(i)
        else:
            A = np.column_stack(test_set)
            rank = np.linalg.matrix_rank(A)

            if rank == len(test_set):
                # Still linearly independent, add this vector
                independent_set.append(v)
                independent_indices.append(i)
            else:
                print(f"Vector v{i+1} = {v} can be expressed as linear combination of previous vectors, skipping")

    print(f"Indices of maximal linearly independent set: {[i+1 for i in independent_indices]}")
    print(f"Maximal linearly independent set: {independent_set}")
    print(f"Rank of maximal linearly independent set: {len(independent_set)}")

    return independent_set, independent_indices

# Test example
print("Finding maximal linearly independent set:")
test_vectors = [
    [1, 2, 3],   # v1
    [2, 4, 6],   # v2 = 2*v1 (linearly dependent)
    [1, 0, 1],   # v3 (possibly independent)
    [0, 1, 1],   # v4 (possibly independent)
    [1, 1, 2]    # v5 = v3 + v4 (linearly dependent)
]

maximal_set, indices = find_maximal_linearly_independent_set(test_vectors)

Relationship Between Linear Dependence and System Solutions

Homogeneous Systems

def analyze_homogeneous_system():
    """
    Analyze the relationship between homogeneous system solutions and linear dependence
    """
    print("Homogeneous system Ax = 0 and linear dependence:")
    print("=" * 50)

    # Column vectors of coefficient matrix A are linearly dependent ⟺ Ax = 0 has non-zero solution
    A1 = np.array([[1, 2, 3],
                   [2, 4, 6],
                   [1, 1, 2]])

    print("Matrix A1:")
    print(A1)

    # Check linear dependence of column vectors
    rank_A1 = np.linalg.matrix_rank(A1)
    print(f"Rank of A1: {rank_A1}")
    print(f"Number of columns in A1: {A1.shape[1]}")

    if rank_A1 < A1.shape[1]:
        print("Column vectors are linearly dependent, homogeneous system has non-zero solution")

        # Find null space
        null_sp = null_space(A1)
        print("Basis of null space:")
        print(null_sp)

        # Verify solution
        if null_sp.size > 0:
            solution = null_sp[:, 0]
            result = A1 @ solution
            print(f"Verification: A1 * {solution} = {result}")
    else:
        print("Column vectors are linearly independent, homogeneous system has only zero solution")

analyze_homogeneous_system()

Non-Homogeneous Systems

def analyze_nonhomogeneous_system():
    """
    Analyze the solution situation of non-homogeneous systems
    """
    print("\nNon-homogeneous system Ax = b:")
    print("=" * 40)

    A = np.array([[1, 2],
                  [2, 4]])  # Rank-deficient matrix
    b1 = np.array([3, 6])   # Case with solution
    b2 = np.array([3, 7])   # Case without solution

    print("Matrix A:")
    print(A)

    # Case 1: Has solution
    print(f"\nCase 1: b = {b1}")
    augmented1 = np.column_stack([A, b1])
    rank_A = np.linalg.matrix_rank(A)
    rank_aug1 = np.linalg.matrix_rank(augmented1)

    print(f"rank(A) = {rank_A}")
    print(f"rank([A|b]) = {rank_aug1}")

    if rank_A == rank_aug1:
        print("Has solution!")
        # Find particular solution
        try:
            x = np.linalg.lstsq(A, b1, rcond=None)[0]
            print(f"A particular solution: {x}")
        except:
            print("Using pseudoinverse to solve")
            x = np.linalg.pinv(A) @ b1
            print(f"A particular solution: {x}")
    else:
        print("No solution")

    # Case 2: No solution
    print(f"\nCase 2: b = {b2}")
    augmented2 = np.column_stack([A, b2])
    rank_aug2 = np.linalg.matrix_rank(augmented2)

    print(f"rank(A) = {rank_A}")
    print(f"rank([A|b]) = {rank_aug2}")

    if rank_A == rank_aug2:
        print("Has solution")
    else:
        print("No solution!")

analyze_nonhomogeneous_system()

Applications of Linear Dependence

Application 1: Span of Vector Sets

def analyze_span():
    """
    Analyze the space spanned by vector sets
    """
    print("Span of vector sets:")
    print("=" * 40)

    # 2D space case
    v1 = np.array([1, 2])
    v2 = np.array([2, 4])  # v2 = 2*v1

    print(f"Vector set: {v1}, {v2}")
    print("These two vectors are linearly dependent, they span a line through the origin")

    # 3D space case
    u1 = np.array([1, 0, 0])
    u2 = np.array([0, 1, 0])
    u3 = np.array([1, 1, 0])  # u3 = u1 + u2

    print(f"\nVector set: {u1}, {u2}, {u3}")
    print("Two of these three vectors are linearly independent, they span the xy-plane")

    # Visualize span
    fig = plt.figure(figsize=(12, 5))

    # 2D case
    ax1 = fig.add_subplot(121)
    t = np.linspace(-3, 3, 100)
    line_points = np.outer(t, v1)

    ax1.plot(line_points[:, 0], line_points[:, 1], 'b-', alpha=0.3, linewidth=3,
             label='span{v1, v2}')
    ax1.arrow(0, 0, v1[0], v1[1], head_width=0.2, head_length=0.2,
              fc='red', ec='red', label='v1')
    ax1.arrow(0, 0, v2[0], v2[1], head_width=0.2, head_length=0.2,
              fc='blue', ec='blue', label='v2=2v1')

    ax1.set_xlim(-4, 4)
    ax1.set_ylim(-4, 4)
    ax1.grid(True, alpha=0.3)
    ax1.legend()
    ax1.set_title('Span of Linearly Dependent Vectors\n(Line)')
    ax1.set_aspect('equal')

    # 3D case projection view
    ax2 = fig.add_subplot(122, projection='3d')

    # Create grid representing plane
    x = np.linspace(-2, 2, 10)
    y = np.linspace(-2, 2, 10)
    X, Y = np.meshgrid(x, y)
    Z = np.zeros_like(X)  # z = 0 plane

    ax2.plot_surface(X, Y, Z, alpha=0.3, color='lightblue')
    ax2.quiver(0, 0, 0, u1[0], u1[1], u1[2], color='red', label='u1')
    ax2.quiver(0, 0, 0, u2[0], u2[1], u2[2], color='green', label='u2')
    ax2.quiver(0, 0, 0, u3[0], u3[1], u3[2], color='blue', label='u3=u1+u2')

    ax2.set_xlim([-2, 2])
    ax2.set_ylim([-2, 2])
    ax2.set_zlim([-1, 1])
    ax2.legend()
    ax2.set_title('Span of Linearly Dependent Vectors\n(Plane)')

    plt.tight_layout()
    plt.show()

analyze_span()

Chapter Summary

graph TD
    A[Linear Dependence] --> B[Definition]
    A --> C[Geometric Meaning]
    A --> D[Determination Methods]
    A --> E[Important Properties]

    B --> F[Non-zero coefficients make linear combination zero]
    C --> G[Collinear/Coplanar/Co-hyperplanar]
    D --> H[Matrix Rank]
    D --> I[Determinant]
    D --> J[Homogeneous System]

    E --> K[Containing zero vector implies dependence]
    E --> L[Extension of dependent set is still dependent]
    E --> M[Subset of independent set is still independent]

    A --> N[Applications]
    N --> O[Maximal Linearly Independent Set]
    N --> P[Structure of System Solutions]
    N --> Q[Dimension of Vector Space]

This chapter explored in depth one of the most core concepts in linear algebra—linear dependence:

Through this chapter, we have deeply understood the dependency relationships between vectors, laying a solid foundation for the next chapter on basis and dimension theory.