SVD Example
>
>
>
>
>
>
#Here is R Code for Exploring SVD's
#First make and view a small data set
X<-matrix(c(1,2,3,4,1,1,2,2,1,2,2,1),nrow=4)
X
[,1] [,2] [,3]
[1,]
1
1
1
[2,]
2
1
2
[3,]
3
2
2
[4,]
4
2
1
>
> #Now make and inspect the SVD of the raw data set
>
> svdX<-svd(X)
> svdX
$d
[1] 6.9146712 1.4055559 0.4601457
$u
[1,]
[2,]
[3,]
[4,]
[,1]
[,2]
[,3]
-0.2399917 -0.2836567 -0.6418278
-0.4144119 -0.5947862 0.6515307
-0.5936900 -0.2448493 -0.3709191
-0.6466830 0.7112082 0.1611953
$v
[,1]
[,2]
[,3]
[1,] -0.7862438 0.45324122 0.4199917
[2,] -0.4534055 0.03861458 -0.8904675
[3,] -0.4198144 -0.89055113 0.1751416
>
>
>
>
>
>
>
>
# svdX$d is a vector with the singular values in it
# svdX$u is the U matrix
# svdX$v is the V matrix
#Verify that the SVD does indeed reproduce the data matrix
svdX$u%*%diag(svdX$d)%*%t(svdX$v)
[,1] [,2] [,3]
[1,]
1
1
1
[2,]
2
1
2
[3,]
3
2
2
[4,]
4
2
1
>
> #Now compute the best rank=1 approximation to the raw data matrix
> #using the the first columns of U and V
>
> svdX$d[1]*(svdX$u[,1]%*%t(svdX$v[,1]))
[,1]
[,2]
[,3]
[1,] 1.304743 0.7524099 0.6966667
[2,] 2.252999 1.2992433 1.2029873
[3,] 3.227666 1.8613070 1.7234100
[4,] 3.515768 2.0274481 1.8772423
1
>
> #Now compute the best rank=2 approximation to the raw data matrix
> #using the first 2 columns of both U and V
>
> svdX$u[,1:2]%*%diag(svdX$d[1:2])%*%t(svdX$v[,1:2])
[,1]
[,2]
[,3]
[1,] 1.124038 0.7370144 1.0517253
[2,] 1.874087 1.2669613 1.9474927
[3,] 3.071683 1.8480178 2.0298926
[4,] 3.968848 2.0660489 0.9870092
>
> #And again, X is computable using all 3 columns of U and V
>
> svdX$u[,1:3]%*%diag(svdX$d[1:3])%*%t(svdX$v[,1:3])
[,1] [,2] [,3]
[1,]
1
1
1
[2,]
2
1
2
[3,]
3
2
2
[4,]
4
2
1
>
> #Now center the X matrix and view the centered version
>
> CX<-X-matrix(c(rep(mean(X[,1]),4),
+
rep(mean(X[,2]),4),rep(mean(X[,3]),4)),nrow=4)
> CX
[,1] [,2] [,3]
[1,] -1.5 -0.5 -0.5
[2,] -0.5 -0.5 0.5
[3,] 0.5 0.5 0.5
[4,] 1.5 0.5 -0.5
>
> #Now compute and inspect the SVD of the centered data matrix
>
> svdCX<-svd(CX)
> svdCX
$d
[1] 2.4142136 1.0000000 0.4142136
$u
[,1] [,2]
[,3]
[1,] -0.6532815 0.5 0.2705981
[2,] -0.2705981 -0.5 -0.6532815
[3,] 0.2705981 -0.5 0.6532815
[4,] 0.6532815 0.5 -0.2705981
$v
[,1]
[,2]
[,3]
[1,] 9.238795e-01 0.000000e+00 -3.826834e-01
[2,] 3.826834e-01 -6.206335e-17 9.238795e-01
[3,] -2.375062e-17 -1.000000e+00 -5.733906e-17
>
> #Verify that the centered data matrix can be reconstructed from the
> #SVD components and compute best low rank approximations the centered
> #data matrix
>
2
> svdCX$u%*%diag(svdCX$d)%*%t(svdCX$v)
[,1] [,2] [,3]
[1,] -1.5 -0.5 -0.5
[2,] -0.5 -0.5 0.5
[3,] 0.5 0.5 0.5
[4,] 1.5 0.5 -0.5
> svdCX$d[1]*(svdCX$u[,1]%*%t(svdCX$v[,1]))
[,1]
[,2]
[,3]
[1,] -1.4571068 -0.6035534 3.745855e-17
[2,] -0.6035534 -0.2500000 1.551584e-17
[3,] 0.6035534 0.2500000 -1.551584e-17
[4,] 1.4571068 0.6035534 -3.745855e-17
> svdCX$u[,1:2]%*%diag(svdCX$d[1:2])%*%t(svdCX$v[,1:2])
[,1]
[,2] [,3]
[1,] -1.4571068 -0.6035534 -0.5
[2,] -0.6035534 -0.2500000 0.5
[3,] 0.6035534 0.2500000 0.5
[4,] 1.4571068 0.6035534 -0.5
> svdCX$u[,1:3]%*%diag(svdCX$d[1:3])%*%t(svdCX$v[,1:3])
[,1] [,2] [,3]
[1,] -1.5 -0.5 -0.5
[2,] -0.5 -0.5 0.5
[3,] 0.5 0.5 0.5
[4,] 1.5 0.5 -0.5
>
> #Put differently, compute the progressively smaller rank=1 components
> #that are added to make the low rank approximations.
>
> #The first component should be the best rank=1 approximation to CX
>
> CX%*%svdCX$v[,1]%*%t(svdCX$v[,1])
[,1]
[,2]
[,3]
[1,] -1.4571068 -0.6035534 3.745855e-17
[2,] -0.6035534 -0.2500000 1.551584e-17
[3,] 0.6035534 0.2500000 -1.551584e-17
[4,] 1.4571068 0.6035534 -3.745855e-17
>
> #Here's the second component
>
> CX%*%svdCX$v[,2]%*%t(svdCX$v[,2])
[,1]
[,2] [,3]
[1,]
0 -3.103168e-17 -0.5
[2,]
0 3.103168e-17 0.5
[3,]
0 3.103168e-17 0.5
[4,]
0 -3.103168e-17 -0.5
>
> #The second component should the difference between the best rank=1
> #and the best rank=2 approximation to CX. Verify this.
>
> (svdCX$u[,1:2]%*%diag(svdCX$d[1:2])%*%t(svdCX$v[,1:2])
+ -svdCX$d[1]*(svdCX$u[,1]%*%t(svdCX$v[,1])))
[,1]
[,2] [,3]
[1,] 2.220446e-16 0.000000e+00 -0.5
[2,] 0.000000e+00 5.551115e-17 0.5
[3,] 0.000000e+00 0.000000e+00 0.5
[4,] -2.220446e-16 0.000000e+00 -0.5
>
3
> #Here's the third component
>
> CX%*%svdCX$v[,3]%*%t(svdCX$v[,3])
[,1]
[,2]
[,3]
[1,] -0.04289322 0.1035534 -6.426871e-18
[2,] 0.10355339 -0.2500000 1.551584e-17
[3,] -0.10355339 0.2500000 -1.551584e-17
[4,] 0.04289322 -0.1035534 6.426871e-18
>
> #Now compute and inspect the sample covariance matrix for the data
>
> C<-(1/4)*(t(CX)%*%CX)
> C
[,1] [,2] [,3]
[1,] 1.25 0.50 0.00
[2,] 0.50 0.25 0.00
[3,] 0.00 0.00 0.25
>
> #Compute and inspect the SVD of the sample covariance matrix
> #Note that U and V are the same
>
> svdC<-svd(C)
> svdC
$d
[1] 1.45710678 0.25000000 0.04289322
$u
[,1] [,2]
[,3]
[1,] -0.9238795
0 -0.3826834
[2,] -0.3826834
0 0.9238795
[3,] 0.0000000
1 0.0000000
$v
[,1] [,2]
[,3]
[1,] -0.9238795
0 -0.3826834
[2,] -0.3826834
0 0.9238795
[3,] 0.0000000
1 0.0000000
>
> #Compute and inspect the eigen decomposition of the sample
> #covariance matrix
>
> eigenC<-eigen(C)
> eigenC
$values
[1] 1.45710678 0.25000000 0.04289322
$vectors
[,1] [,2]
[,3]
[1,] 0.9238795
0 -0.3826834
[2,] 0.3826834
0 0.9238795
[3,] 0.0000000
1 0.0000000
>
> #The (column) eigenvectors are in eigenC$vectors and the eigenvectors
> #are in eigenC$vectors
>
4
> #Note that the eigenvectors of the sample covariance matrix are
> #plus-or-minus the columns of V from the SVD of C and the eigenvalues
> #of C are the singular values from the SVD of C
>
> #Note that since the sample size for the original data set is 4
> #(CX)'CX is 4*C, so that 4 times the eigenvalues of the sample
> #covariance matrix C are the eigenvalues and thus singular values of
> #(CX)'CX. These are the squares of the singular values of the
> #centered data matrix, CX.
>
> #Verify the relationship just stated.
>
> sqrt(4*svdC$d)
[1] 2.4142136 1.0000000 0.4142136
>
> svdCX$d
[1] 2.4142136 1.0000000 0.4142136
>
> #Compute best low rank approximations to the sample covariance matrix
> #First, the rank=1 approximation
>
> svdC$d[1]*(svdC$u[,1]%*%t(svdC$v[,1]))
[,1]
[,2] [,3]
[1,] 1.243718 0.5151650
0
[2,] 0.515165 0.2133883
0
[3,] 0.000000 0.0000000
0
>
> #Now the best rank=2 approximation
>
> svdC$u[,1:2]%*%diag(svdC$d[1:2])%*%t(svdC$v[,1:2])
[,1]
[,2] [,3]
[1,] 1.243718 0.5151650 0.00
[2,] 0.515165 0.2133883 0.00
[3,] 0.000000 0.0000000 0.25
5