Number of Basis Functions log 10 tr K K approx

! "#$%#!&'$(

) ! ! *

+ ,) -*

! *

. .

! " /

0 !.

!

0" 0

(*1 (

2 ! 1

"

! !

3 ! 43) %5567 +*

,) 4# " %5567 # 1*

4/ %5587 ) !"

# * 9 :9 : *

!!

+!+)

47;

%

9

9

::<

=

9%:

$ ) $

9($: 4+> %567

! *

= + ?

Ê +!*

,9

:

"

( /! 4%5@%7 0

AB 4%557 ! , ,! ! ! *

+! 9%:"

?

9:;

9

: 9:

!!!,!9:-*

$ ! -

*

" ! - !

!! "

/!9:,! 9!

. ! *

-0

?;9

:.

: " $

! -! +

Æ

09:" 3 *

! !

99::!! *

9

:

,! ! ! 9 - : 4 !C)

"%5567 !43)

"%55@7" $, !!*

4%5587"

! ! 9!:+*

+! !0 *

- 4%5D6!"%555#

%5587" $, ! ! *

!,! 9: !0

?; 9

: 4 7 94 7 ?; % :"

! ,! 9

: 90 *

: ! !

!"

-!,,!

99::;

%

99::

9':

D,!,!,E

! !. "

!%9':! ! ($

9

:9

:

;9

: 9F:

!9%: 4#"%5567

Ê

4 7;

%

<

=

96:

! 96:

;9

< =:

;9< =:

9D:

;

9,! ! ! )*

$ : )" !,*

, )

;

4 7 ! +

,

Ê

4 7;

%

<

=

$ Ê

!0

;

;

" !

;9

< =

:

9@:

,! !9 <

:!!9

:

"

! "

! ! ! !, !

" ! - *

!" A! ! !

!%" $,

!!9 :

" !! !

! 0 ?

! ! 0 ! ! G

;

! !

-

9

!

> ! :"

!, -

G

,) 9 ! *

): ! ! ! !

0 G

+"!!

2

G

?;

9

G

:

98:

!! 0

,! ;H ,!

;9

:;9

H

H :;9H

:

95:

! + 98:+! *

9! :

G

?; 0

9

G

:

9%:

I), ,! , ,!

! ! ! 0 ,

,+!

-0 G

" &-

!!.+!%* !

! 0"

D"F ! ,! *

0 G

0 !

" ,,) , -

!,! !

9

: 9 :" ,)

!!!.!

9

: ! 0

."

# $

! !+ ,"

! 0 !"

F ,! , -

!

' !+

! " 0*

6" 2

D ! ! !

! " ! *

!9 (2*

4"%5557: @"

! ,

!! !*

0

! 9

:*

!

;

9

:"

0 0 Æ

0 ! ""

G

;

9%%:

!

G

! !0"

"

, , ) - ! !

Æ

!!!0

9%%:"

" % & '

2 *

!0

" *

)!

! ! ! 4 J!

%55'>%556 !C) "%5557!,

,!!E !,0

0 ""

9 :"

$ , ,? 9%:-!0K

! 4 7 ! *

L ! 0 !

L

" 9:-

G

"" -*

0

" 9':!

G

!, "

&( (

9%: 0

0 " ! *

!

,

9 :9,! ! ) ! *

! : ! , *

9

:" ! ! ,! ,,

"

! )

-0+ )! K

! ! ! 0! !

4 7" !

! ,?

) " ' (

!

" # !!"

! ! 9

: !!"

!! ?;0

99::

! 4%7

0

" !

! ,!

56 ! 6

+ 656 ; 65 , *

½

¾

!,! "

& %

!, !

009!%:"

% 001

*

*

;; ;

<<

, 4 7

; 9:

;

; 9:

*

/ ,!! !

, , 9

! ! 1

!, ! :" ,

#&2 ),!!

;6"9"!

! M 6 !

:" 2 % !, ! + 65

! " !

, ! 1

! " !, !

! ! ! *

* ,! ! ""

!2'" !E !,

) !

! ,"

! + % (

; G

;

9

G

:

9%:

"" ! 9

:" !

0!,!, ,

9 , 9 :!:"

, 9%:

?;

9

G

:

!

, !

9%: !

" !

! !2 ,,

!

G

":" !

,

9 " F:"

! , !

$E4%5D'7"

, -.../

$ !

! " % "

4##<$7# "

;

& "

#

! "

# %? %

% '"

% ;

1

1

&

9%':

$ , - ,! ! -

9%':,! !*

! *

" A!, ,! ,

!

!!!,-

! "

2 ! !. *

,! ! !0

! 0 "

! !, ! !

!! 0

9 :99::"

I ! ! *

, )K !0 L

" ,,

! ! !

L "

,

! ! ,!,! 0*

! 2 - Æ

! ! G

+! "

2 ,!!-9;

%:" $ ,!!9 G

;

:

9%F:

;

9%6:

.

; " !

, 9

G

:4 7

"

! )! !*

! ! 9! > *

$ :" $ *

.Æ

9%F: ,! !

G

! !

!,, ,!

9

G

:

47 9%D:

. " !

9

: 9 ! !+

*:" / !

! ! 9 9

:

.:" !!,9

:!*

, ! !

" ! 0!

,! ,, "

0

!+

!! " ! !!

9

:9

: ; 9

: 9%:

! ! *

" ,0*

! ! !($

!*

,!" 2,!

! ! 0 0 G

! 0

G

-*

"

!! ! ,

, ! !2

!" &,,0 *

!%* 9 : " !

,) , ! - *

"

G

?;

9%@:

! ;% ,! G

;

" ,,

+

G

;

9%8:

, '"%

;

;

9%5:

! !!*

! G

," ! !

9

G

:4 7

47 9:

! G

! ." !

! ,

9

G

: 9

G

: 9 :?

G

, ! !!*

+ , 9:

G

G

;

G

9%:

!! !0

G

-"

, !

G

9,! !

! !:

9 G

:;

G

9:

,! ! 9 : *

!!+"

1 "

9%6: 9%5:! !

L ) ! *

- !, > "

! ! * ,

!) ! - ? -

! ,, L

!( )""

?;Æ

9':

! - !,

> ? !*

K ! ! *

L !" A ! ) ! !

.-!0 !

, !

!

!

;Æ

"

! . ,

! ! !

"

! ,! ! !

E,!. !

!

! " ! ,

! !

A ) !

!

! + G

,! ! !

G

, ! ! ! 9%D: !"

> !!!

, ,9 -0K:

G

9F:

;

G

'

9

G

:

96:

,!

'

;

G

G

G

9D:

!! !

G

G

9@:

;

G

G

G

! ! 9@:

K 9! ++:"

! 0 9@:9

: !

! 9

:!,"

! 0

, ! !

!,,!

, 0" 9F:

,+

G

'

9

G

:

98:

,!

'

;

G

G

G

95:

! !

G

G

9':

;

G

G

G

;

G

G

9'%:

,!9'%: , !!,!

G

!0 G

" *

9'%:9 :! !!

! *

!* ( 2 4"

%5557" ! ! ! !

!

!"

! 0! !!

L ,!!* !

""

K;0

G

9':

! ! !

! " ! ! 9,! !

9

: !!:!! 9':

9 :" /! !! !

""9 :"

!!

! 9 F"%

F":" ! !

,! ! +!,

! E ,

"

/! !2 !

- ! ,

E !

- ! G

G

,! !

"" 1 !*

43%55F7"

!

# 3

!!,!

+ ,) , ) *

+ 9 6 :

! *

! 0" A ! ,

! !, *

,!! "

/ ! ! !

! 4&) " %5587

! 1 !, " !

- +6D9'6%6D:!

+ F%@@9'%%@@

:" !- , ! ,

+ *,,!

! 1 !" 2 !

9L L :,

9%:9%:9%:" /

!"

9

:;&

,!! 9'':

/!";%= ;% !

!" ; %= ; % !

" !,+*

! "" %6%"

# %

2 , 0

> !,) 9*

!:" 9 :9!!

9

: !2: 0,

!"

% 02

*

;; ;

;

!0

<<

, 4 7

( (

;

,!4 7

G

;

,!

0 "

;

;K;0

&4

';9

9

:

:

9

%:

;

G

;<

' )%

;9

:

;

*

# (

! - ! ! ) ,!! !

" , " /

!1! !

" ! , !

;6" ,! "

! !"

¾

0 50 100 150 200 250 300 350 400 450 500 0

0.5 1 1.5 2 2.5 3 3.5 4

Number of Basis Functions log 10 tr K K approx

Size of the Residuals for the Abalone dataset subset size 1, 2, 5, 10, 20, 50, 59, 100, 200, 500 (top to bottom)

! "

#

$ !%&'%(&(

'(')%((&(( * !'(

+

0 50 100 150 200 250 300 350 400 450 500

1 1.5 2 2.5 3 3.5 4

4.5 Regularized Risk and Expected Risk on the Abalone Dataset (Eigensystem)

Subspace Dimension

Risk

Regularized Risk Test Error

0 50 100 150 200 250 300 350 400 450 500

1 1.5 2 2.5 3 3.5 4

4.5 Regularized Risk and Expected Risk on the Abalone Dataset (Sparse Decomposition)

Subspace Dimension

Risk

Regularized Risk Test Error

!, -

2%!,! +65!

" !

,!1"

0 50 100 150 200 250 300 350 400 450 500

6 5 4 3 2 1 0 1 2 3 4

Number of Basis Functions log 10 tr K K approx

Size of the Residuals for the Abalone dataset

2  ² / d = 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10 (top to bottom)

0 50 100 150 200 250 300 350 400 450 500

6 5 4 3 2 1 0 1 2 3 4

Number of Dimensions log 10 Sum j=i m  j

Size of the residual trace for the Abalone dataset

2  ² / d = 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10 (top to bottom)

! "

#

.

/ .

#! % 5 6 5

/ ! , ! !?

+ ,) ,!

9D: *

,! ! 1

0 +! *

" = ! , !

+!) ! )

!,

"

0 " / 1

2 * )%(

2%(

Æ+

, 1

0" A !!

E "

! 0

!,! ! 99 :!!

9

: :

91!E *

:

.9 :!!9

:"

)N!.! !

+ 92

6:" ! 2 '?

! !, !*

! !0

0! !"

$

,F%,!

+ 9

: !

- " ! + !

,! 9!

Æ :"

"

/!!0 !$ *

. ( ; ,

! , ,

0 ( !> " "

# 0 ! 0

0 !, 0*

,!

! !.

$ 4 !C) "%5557" 2!

0 20 40 60 80 100 120 140 160 180 200

6 5 4 3 2 1 0 1 2 3

Number of Basis Functions log 10 tr K K approx

Size of the Residuals for the Abalone dataset

top to bottom: 2  ² / d = 1 2 5 10 20 50

0 20 40 60 80 100 120 140 160 180 200

0 5 10 15 20 25 30 35

40 Regularized Risk and Expected Risk on the Boston Housing Dataset (Eigensystem)

Subspace Dimension

Risk

Regularized Risk Test Error

0 20 40 60 80 100 120 140 160 180 200

0 5 10 15 20 25 30 35

40 Regularized Risk and Expected Risk on the Boston Housing Dataset (Sparse Decomposition)

Subspace Dimension

Risk

Regularized Risk Test Error

! "

#

-

!

, -

*

" &!0 ,!!),

0 !0 "

7 ' "

1 . !

. 3 *

!" /! ! ,

!.!!

0" *

0

?; <

! ! ! !9!*

((34%55F7:" !!

+"

/! ! *

4/"%5587! 0

0!!" $, ,

!0 G

?;

G

< ! ! !

9

:" !

! ! 9 :

G

)0" 2!

) ! 0

9

:"

!, ! ! !"

7 ) &% %"

- !+!*

+) *

! 4%5D67" /!! ! *

+!%*

! 0 Æ

!+0 "

!

,! 9,! ! ! 0 ! 0

,! :, -

!!

! ! "

! !

1IO ! * ,! ! *

! Æ

" ! !

, !? -

0 ! M

!0 . Æ !! *

!+ ! G

9""

- :!,

"

¿

,3 4

1> !. *

9 : !! !!

!.!! 9 """4#*

" %5557:" !,) >

, ! !!

!! . ! *

! "

!! !)!

"

- 0 9""

,!0 . :

,!P

! - ! "

9 4Q)) $%5557

. ,): ! *

." R !! !

! .! *

"

0 0 !,

! ! > *

> , 9"" 55S 0*

! >

' ):" / 0 ! E

! + " *

. 0 0 !

! !

!!* "

7! 0

! 4)+ %55D7!,! *

0 0"

0*

!4>%5567

! . !*

0 !" !

!,! !!!

" ! ! >

! !"

! ! *

! *

"" !" !

- 0 !0

" ! !.

-0*

,!! , )

9 ! .,0:

,!!0 !,

!"

8 10

! ! 4 " %5557

! !.!

" E !*

+ ! !

> ! ! !!

9 F"': 1> *

1* ! )

! "

8 , %

! -

! &2*) 9

"" 4 !C) " %55@7:? ! *

!

" &

0 !9""K

!!

!3 :K

! !!!!E

00 G

" $,! !

> !!

!

! - !" 2

)! ! &2 ! !.

"

$ %

/ ! 0 *

-

" ! *

! !! 9 :

,! ! - ,! *

! 9!

.9

: 0:"

20!,!!!!

!0 ! ,! !

, ! !

)!" /!!

0 , +

0 ! ,! ! , *

D"

9% ! ! ,

! !2 ! ! 9

DL%*%: ! ! " !

! !) 1 & ) $ A

& / ! *

" +>" ! )"

* & D8?''@TFF%56"

" &) " (! " Q" +" *

! %558"

!?LL,,," " "LLI"!"

"! " !"" *

"+!

,!9%:?''TD%%555"

"02"AB" *

+)! " *

%8?%D@DT%D56 %55"

"#1")"," !

!!!!"-

./0$1, ! %555"

2"#" . ,0*

!" 2! ,!

%9D:?%F66T%F8%558"

2" # " Q " 1" +

! ,) ! " 2!

,!@?%5TD5%556"

/" $E" 1 .

" +! *

*68?%'T'%5D'"

""Q)) "$"0

-"""(

" " " " ! *"

2! 3 - *

%555"1"

#" " ( #" /!"

,& !

! " * & ?

F56T6 %5@%"

R" )+" 0 *

,)" +! * ( 86?58T

%5%55D"

" J" J!" ! 1 *

. " 3)))

- F%?''5@T'F%6%55'"

A" I" " I

" 4

%'?FFFTF6%5D6"

&"

G

(">" 0 *

" 3*& +! ,! F9:?

@T'F %556"

&" !C) " & 3" 3)" 0

)" ""2

"!- % 5 3

, # " 6

" C #" C ! " "

" )* !" 3)))

2! 2# %96:?%T%%@

%555"

&" !C) (""&2"#1"

"13"3)" *

! ,! )

-" 3))) -

F6?@68T @D6%55@"

Q")" &(!) 1&"

1%588"

"Q""$ # "1! ! !*

!C&%558"

" Q" A" I" &" !C) "

) " ! 55*

' /

%555"

" Q" 3" IAUA?

. " A*5F*%6 *

A !1 "Q

%55F"

3"3)" 2! $ "

"R"%556" &*'8@*5F665*8"

3"3)"#, !""

! 0*

""+"Q*

" 1 ! *" 2!

3 - 7 8%T8@

%55@"1"

"(""/" 1 ,! ?

2 *

" $ 3 8 &

%558"

""/"Q"&" !C) "#*

+ + ,)

!

" AI**58*%5

$,%558"