Update Nov2019

Author: AdrienTaylor

Examples/01_Methods for unconstrained convex minimization/A_ProximalPointMethod.m

@@ -0,0 +1,60 @@
+clear all; clc;
+% In this example, we use a proximal point method for solving the 
+% non-smooth convex minimization problem
+%   min_x F(x); for notational convenience we denote xs=argmin_x F(x);
+%
+% We show how to compute the worst-case value of F(xN)-F(xs) when xN is
+% obtained by doing N steps of a proximal method starting with an initial
+% iterate satisfying ||x0-xs||<=1.
+%
+% Alternative interpretations:
+% (1) the following code compute the solution to the problem
+%       max_{F,x0,...,xN,xs} (F(xN)-F(xs))/||x0-xs||^2 
+%            s.t. x1,...,xN are generated via the proximal point method,
+%                 F is closed, proper, and convex.
+%     where the optimization variables are the iterates and the convex
+%     function F.
+%
+% (2) the following code compute the smallest possible value of 
+%     C(N, step sizes) such that the inequality
+%       F(xN)-F(xs)  <= C(N, step sizes) * ||x0-xs||^2
+%     is valid for any closed, proper and convex F and any sequence of
+%     iterates x1,...,xN generated by the proximal point method on F.
+%
+
+
+% (0) Initialize an empty PEP
+P = pep();
+
+% (1) Set up the objective function
+F = P.DeclareFunction('Convex');      % F is the objective function
+
+% (2) Set up the starting point and initial condition
+x0      = P.StartingPoint();        % x0 is some starting point
+[xs,fs] = F.OptimalPoint();         % xs is an optimal point, and fs=F(xs)
+P.InitialCondition( (x0-xs)^2 <= 1);% Initial condition ||x0-xs||^2 <= 1
+
+% (3) Algorithm
+N   = 10;		% number of iterations
+gam = @(k)(1);	% step size (possibly a function of k)
+
+x = x0;
+for i = 1:N
+    x = proximal_step(x, F, gam(i));
+end
+xN = x;
+
+% (4) Set up the performance measure
+fN = F.value(xN);
+P.PerformanceMetric(fN-fs);
+
+% (5) Solve the PEP
+P.solve()
+
+% (6) Evaluate the output
+double(fN-fs)   % worst-case objective function accuracy
+
+% The result should be (and is) 1/(4*\sum_{i=1}^N gam(i))
+% see Taylor, Adrien B., Julien M. Hendrickx, and François Glineur.
+%     "Exact Worst-case Performance of First-order Methods for Composite
+%     Convex Optimization.", SIAM Journal on Optimization (2017)

Examples/01_Methods for unconstrained convex minimization/B_FastProximalPointMethod.m

@@ -0,0 +1,78 @@
+clear all; clc;
+% In this example, we use the fast proximal point method of Guler [1] for
+% solving the non-smooth convex minimization problem
+%   min_x F(x); for notational convenience we denote xs=argmin_x F(x);
+%
+% [1] O. Güler. New proximal point algorithms for convex minimization.
+%               SIAM Journal on Optimization, 2(4):649–664, 1992.
+% 
+% We show how to compute the worst-case value of F(xN)-F(xs) when xN is
+% obtained by doing N steps of the method starting with an initial
+% iterate satisfying f(x0)-f(xs)+A/2*||x0-xs||^2<=1 for some A>0.
+%
+% Alternative interpretations:
+% (1) the following code compute the solution to the problem
+%       max_{F,x0,...,xN,xs} (F(xN)-F(xs))/( f(x0)-f(xs)+A/2*||x0-xs||^2 ) 
+%            s.t. x1,...,xN are generated via Guler's method,
+%                 F is closed, proper, and convex.
+%     where the optimization variables are the iterates and the convex
+%     function F.
+%
+% (2) the following code compute the smallest possible value of 
+%     C(N, step sizes) such that the inequality
+%       F(xN)-F(xs)  <= C(N, step sizes) * ( f(x0)-f(xs)+A/2*||x0-xs||^2 )
+%     is valid for any closed, proper and convex F and any sequence of
+%     iterates x1,...,xN generated by Guler's method on F.
+%
+
+
+% (0) Initialize an empty PEP
+P = pep();
+
+% (1) Set up the objective function
+F = P.DeclareFunction('Convex');      % F is the objective function
+
+% (2) Set up the starting point and initial condition
+x0      = P.StartingPoint();        % x0 is some starting point
+[xs,fs] = F.OptimalPoint();         % xs is an optimal point, and fs=F(xs)
+
+% (3) Algorithm
+N       = 13;		% number of iterations
+A0      = 10;        % initial value of A  (see paper)
+lambda  = @(k)(k/1.1);  % stepsizes (possibly function of k)  (see paper)
+
+A   = cell(N+1,1); A{1}=A0;
+
+x = x0;
+v = x0; % second sequence of iterates (see paper)
+for i = 1:N
+    alpha  = (sqrt( (A{i}*lambda(i))^2 + 4 * A{i}*lambda(i) ) - A{i}*lambda(i)) / 2;
+    y      = (1-alpha) * x + alpha * v;
+    x      = proximal_step(y, F, lambda(i));
+    v      = v + 1/alpha * (x - y);
+    A{i+1} = (1-alpha) * A{i};
+end
+xN = x;
+
+f0 = F.value(x0);
+P.InitialCondition( f0-fs+A{1}/2*(x0-xs)^2 <= 1); % Initial condition 
+
+% (4) Set up the performance measure
+fN = F.value(xN);
+P.PerformanceMetric(fN-fs);
+
+% (5) Solve the PEP
+P.solve()
+
+% (6) Evaluate the output
+% The result should be better than the theoretical guarantee from [1]:
+% f(xN)-f(xs) <= 4/A{1}/(sum_{i=1}^N sqrt(lambda(i)))^2 * ( f0-fs+A{1}/2*(x0-xs)^2 )
+%
+% comparison:
+accumulation = 0;
+for i = 1:N
+    accumulation=accumulation+sqrt(lambda(i));
+end
+theoretical_guarantee = 4/A{1}/accumulation^2;
+pesto_guarantee       = double(fN-fs);
+fprintf('Theoretical guarantee from [1]: f(xN)-f(xs)<= %6.5f * ( f0-fs+A{1}/2*(x0-xs)^2 )\n \t guarantee from pesto:  f(xN)-f(xs)<= %6.5f * ( f0-fs+A{1}/2*(x0-xs)^2 )\n',theoretical_guarantee,pesto_guarantee)

Examples/SubgradientMethod.m renamed to Examples/01_Methods for unconstrained convex minimization/C_SubgradientMethod.m

@@ -10,24 +10,24 @@
 % iterate satisfying ||x0-xs||<=1.
 
 % (0) Initialize an empty PEP
-P=pep();
+P = pep();
 
 % (1) Set up the objective function
-param.R=1;	% 'radius'-type constraint on the subgradient norms: ||g||<=1
+param.R = 1;	% 'radius'-type constraint on the subgradient norms: ||g||<=1
 
 % F is the objective function
-F=P.DeclareFunction('ConvexBoundedGradient',param); 
+F = P.DeclareFunction('ConvexBoundedGradient',param); 
 
 % (2) Set up the starting point and initial condition
-x0=P.StartingPoint();            % x0 is some starting point
-[xs,fs]=F.OptimalPoint();        % xs is an optimal point, and fs=F(xs)
-P.InitialCondition((x0-xs)^2<=1);% Add an initial condition ||x0-xs||^2<= 1
+x0      = P.StartingPoint();         % x0 is some starting point
+[xs,fs] = F.OptimalPoint();          % xs is an optimal point, and fs=F(xs)
+P.InitialCondition( (x0-xs)^2 <= 1); % Initial condition ||x0-xs||^2<= 1
 
 % (3) Algorithm and (4) performance measure
-N=5; % number of iterations
-h=ones(N,1)*1/sqrt(N+1); % step sizes
+N = 5;                     % number of iterations
+h = @(k)(1/sqrt(N+1)); % step sizes
 
-x=x0;
+x = x0;
 
 % Note: the worst-case performance measure used in the PEP is the 
 %       min_i (PerformanceMetric_i) (i.e., the best value among all
@@ -36,24 +36,24 @@
 
 % we create an array to save all function values (so that we can evaluate
 % them afterwards)
-f_saved=cell(N+1,1);
+f_saved = cell(N+1,1);
 for i=1:N
-    [g,f]=F.oracle(x);
-    f_saved{i}=f;
+    [g,f]       = F.oracle(x);
+    f_saved{i}  = f;
     P.PerformanceMetric(f-fs);
-    x=x-h(i)*g;
+    x = x - h(i) * g;
 end
 
-[g,f]=F.oracle(x);
-f_saved{N+1}=f;
+[g,f]        = F.oracle(x);
+f_saved{N+1} = f;
 P.PerformanceMetric(f-fs);
 
 % (5) Solve the PEP
 P.solve()
 
 % (6) Evaluate the output
 for i=1:N+1
-    f_saved{i}=double(f_saved{i});
+    f_saved{i} = double(f_saved{i});
 end
 f_saved
 % The result should be 1/sqrt(N+1).

Examples/01_Methods for unconstrained convex minimization/D_SubgradientMethod_ExactLineSearch.m

@@ -0,0 +1,53 @@
+clear all; clc;
+% In this example, we use a subgradient method with exact line search for
+% solving the non-smooth convex minimization problem
+%   min_x F(x); for notational convenience we denote xs=argmin_x F(x);
+% where F(x) satisfies a Lipschitz condition; i.e., it has a bounded
+% gradient ||g||<=R for all g being a subgradient of F at some point.
+%
+% The method originates from:
+% [1] Y. Drori and A. Taylor. Efficient first-order methods for convex
+% minimization: a constructive approach. Mathematical Programming (2019)
+%
+% We show how to compute the worst-case value of F(xN)-F(xs) when xN is
+% obtained by doing N steps of the specific subgradient method starting
+% with an initial iterate satisfying ||x0-xs||<=1.
+
+% (0) Initialize an empty PEP
+P = pep();
+
+% (1) Set up the objective function
+param.R = 1;	% 'radius'-type constraint on the subgradient norms: ||g||<=1
+
+% F is the objective function
+F = P.DeclareFunction('ConvexBoundedGradient',param); 
+
+% (2) Set up the starting point and initial condition
+x0      = P.StartingPoint();         % x0 is some starting point
+[xs,fs] = F.OptimalPoint();          % xs is an optimal point, and fs=F(xs)
+P.InitialCondition( (x0-xs)^2 <= 1); % Initial condition ||x0-xs||^2<= 1
+
+% (3) Algorithm and (4) performance measure
+N = 5;                     % number of iterations
+x = cell(N+1,1);
+g = cell(N+1,1);
+f = cell(N+1,1);
+
+x{1}        = x0;
+[g{1},f{1}] = F.oracle(x{1});
+d           = g{1};
+for i=1:N
+    y                        = i/(i+1) * x{i} + 1/(i+1) * x{1};
+    [x{i+1}, g{i+1}, f{i+1}] = exactlinesearch_step(y,F,d);
+    d                        = d + g{i+1};
+end
+
+P.PerformanceMetric(f{N+1}-fs);
+
+% (5) Solve the PEP
+P.solve()
+
+% (6) Evaluate the output
+double(f{N+1}-fs)
+
+% The result should be 1/sqrt(N+1), see [1].

Examples/GradientMethod.m renamed to Examples/01_Methods for unconstrained convex minimization/E_GradientMethod.m

@@ -6,43 +6,44 @@
 % We show how to compute the worst-case value of F(xN)-F(xs) when xN is
 % obtained by doing N steps of the gradient method starting with an initial
 % iterate satisfying ||x0-xs||<=1.
-
+%
+% Result to be compared with that of
+% [1] Yoel Drori. "Contributions to the Complexity Analysis of
+%     Optimization Algorithms." PhD thesis, Tel-Aviv University, 2014.
 
 % (0) Initialize an empty PEP
-P=pep();
+P = pep();
 
 % (1) Set up the objective function
-param.mu=0;	% Strong convexity parameter
-param.L=1;      % Smoothness parameter
+param.mu = 0;     % Strong convexity parameter
+param.L  = 1;     % Smoothness parameter
 
 F=P.DeclareFunction('SmoothStronglyConvex',param); % F is the objective function
 
 % (2) Set up the starting point and initial condition
-x0=P.StartingPoint();		 % x0 is some starting point
-[xs,fs]=F.OptimalPoint(); 		 % xs is an optimal point, and fs=F(xs)
-P.InitialCondition((x0-xs)^2<=1); % Add an initial condition ||x0-xs||^2<= 1
+x0      = P.StartingPoint();		 % x0 is some starting point
+[xs,fs] = F.OptimalPoint(); 		 % xs is an optimal point, and fs=F(xs)
+P.InitialCondition( (x0-xs)^2 <= 1); % Initial condition ||x0-xs||^2<= 1
 
 % (3) Algorithm
-h=1/param.L;		% step size
-N=10;		% number of iterations
-
-x=x0;
-for i=1:N
-    x=x-h*F.gradient(x);
-    % % Alternate - shorter - form:
-    % x=gradient_step(x,F,gam);
+h = 1/param.L;		% step size
+N = 10;             % number of iterations
+
+x = x0;
+for i = 1:N
+    x = x-h*F.gradient(x);
 end
-xN=x;
+xN = x;
 
 % (4) Set up the performance measure
-fN=F.value(xN);                % g=grad F(x), f=F(x)
-P.PerformanceMetric(fN-fs); % Worst-case evaluated as F(x)-F(xs)
+fN = F.value(xN);
+P.PerformanceMetric(fN-fs);      % Worst-case evaluated as F(x)-F(xs)
 
 % (5) Solve the PEP
 P.solve()
 
 % (6) Evaluate the output
 double(fN-fs)   % worst-case objective function accuracy
 
-% The result should be
+% The result should be (see [1])
 % param.L/2/(2*N+1)

Examples/GradientExactLineSearch.m renamed to Examples/01_Methods for unconstrained convex minimization/F_GradientExactLineSearch.m

@@ -13,38 +13,40 @@
 % obtained by doing N steps of the method starting with an initial
 % iterate satisfying F(x0)-F(xs)<=1.
 %
-% The full approach (based on convex relaxations) is available in 
+% The detailed approach (based on convex relaxations) is available in 
 %  De Klerk, Etienne, François Glineur, and Adrien B. Taylor. 
 %  "On the worst-case complexity of the gradient method with exact 
 %  line search for smooth strongly convex functions." 
-%  Optimization Letters (2016).
+%  Optimization Letters (2017).
 
 % (0) Initialize an empty PEP
-P=pep();
+P = pep();
 
 % (1) Set up the objective function
-param.mu=.1;	% Strong convexity parameter
-param.L=1;      % Smoothness parameter
+param.mu = .1;	% Strong convexity parameter
+param.L  = 1;      % Smoothness parameter
 
-F=P.DeclareFunction('SmoothStronglyConvex',param); 
+F = P.DeclareFunction('SmoothStronglyConvex',param); 
 % F is the objective function
 
 % (2) Set up the starting point and initial condition
-x0=P.StartingPoint();		 % x0 is some starting point
-[xs,fs]=F.OptimalPoint(); 		 % xs is an optimal point, and fs=F(xs)
-[g0,f0]=F.oracle(x0);
-P.InitialCondition(f0-fs<=1); % Add an initial condition f0-fs<= 1
+x0      = P.StartingPoint();		 % x0 is some starting point
+[xs,fs] = F.OptimalPoint(); 		 % xs is an optimal point, and fs=F(xs)
+[g0,f0] = F.oracle(x0);              
+P.InitialCondition(f0-fs<=1);        % Initial condition f0-fs<= 1
 
 % (3) Algorithm
 N = 2;
 x = x0;
 for i = 1:N
     g = F.gradient(x);
+    
+    % exact line search on F from point x and in direction g:
     x = exactlinesearch_step(x,F,g);
 end
 
 % (4) Set up the performance measure
-[g,f]=F.oracle(x);
+[g,f] = F.oracle(x);
 P.PerformanceMetric(f-fs); % Worst-case evaluated as F(x)-F(xs)
 
 % (5) Solve the PEP