[ VIGRA Homepage | Function Index | Class Index | Namespaces | File List | Main Page ]

quadprog.hxx |

1 /************************************************************************/
2 /* */
3 /* Copyright 2008 by Ullrich Koethe */
4 /* */
5 /* This file is part of the VIGRA computer vision library. */
6 /* The VIGRA Website is */
7 /* http://hci.iwr.uni-heidelberg.de/vigra/ */
8 /* Please direct questions, bug reports, and contributions to */
9 /* ullrich.koethe@iwr.uni-heidelberg.de or */
10 /* vigra@informatik.uni-hamburg.de */
11 /* */
12 /* Permission is hereby granted, free of charge, to any person */
13 /* obtaining a copy of this software and associated documentation */
14 /* files (the "Software"), to deal in the Software without */
15 /* restriction, including without limitation the rights to use, */
16 /* copy, modify, merge, publish, distribute, sublicense, and/or */
17 /* sell copies of the Software, and to permit persons to whom the */
18 /* Software is furnished to do so, subject to the following */
19 /* conditions: */
20 /* */
21 /* The above copyright notice and this permission notice shall be */
22 /* included in all copies or substantial portions of the */
23 /* Software. */
24 /* */
25 /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND */
26 /* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES */
27 /* OF MERCHANTABILITY, FITNESS FOR activeSet PARTICULAR PURPOSE AND */
28 /* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT */
29 /* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, */
30 /* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING */
31 /* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR */
32 /* OTHER DEALINGS IN THE SOFTWARE. */
33 /* */
34 /************************************************************************/
35
36 #ifndef VIGRA_QUADPROG_HXX
37 #define VIGRA_QUADPROG_HXX
38
39 #include <limits>
40 #include "mathutil.hxx"
41 #include "matrix.hxx"
42 #include "linear_solve.hxx"
43 #include "numerictraits.hxx"
44 #include "array_vector.hxx"
45
46 namespace vigra {
47
48 namespace detail {
49
50 template <class T, class C1, class C2, class C3>
51 bool quadprogAddConstraint(MultiArrayView<2, T, C1> & R, MultiArrayView<2, T, C2> & J, MultiArrayView<2, T, C3> & d,
52 int activeConstraintCount, double& R_norm)
53 {
54 typedef typename MultiArrayShape<2>::type Shape;
55 int n=columnCount(J);
56 linalg::detail::qrGivensStepImpl(0, subVector(d, activeConstraintCount, n),
57 J.subarray(Shape(activeConstraintCount,0), Shape(n,n)));
58 if (abs(d(activeConstraintCount,0)) <= NumericTraits<T>::epsilon() * R_norm) // problem degenerate
59 return false;
60 R_norm = std::max<T>(R_norm, abs(d(activeConstraintCount,0)));
61
62 ++activeConstraintCount;
63 // add d as a new column to R
64 columnVector(R, Shape(0, activeConstraintCount - 1), activeConstraintCount) = subVector(d, 0, activeConstraintCount);
65 return true;
66 }
67
68 template <class T, class C1, class C2, class C3>
69 void quadprogDeleteConstraint(MultiArrayView<2, T, C1> & R, MultiArrayView<2, T, C2> & J, MultiArrayView<2, T, C3> & u,
70 int activeConstraintCount, int constraintToBeRemoved)
71 {
72 typedef typename MultiArrayShape<2>::type Shape;
73
74 int newActiveConstraintCount = activeConstraintCount - 1;
75
76 if(constraintToBeRemoved == newActiveConstraintCount)
77 return;
78
79 std::swap(u(constraintToBeRemoved,0), u(newActiveConstraintCount,0));
80 columnVector(R, constraintToBeRemoved).swapData(columnVector(R, newActiveConstraintCount));
81 linalg::detail::qrGivensStepImpl(0, R.subarray(Shape(constraintToBeRemoved, constraintToBeRemoved),
82 Shape(newActiveConstraintCount,newActiveConstraintCount)),
83 J.subarray(Shape(constraintToBeRemoved, 0),
84 Shape(newActiveConstraintCount,newActiveConstraintCount)));
85 }
86
87 } // namespace detail
88
89 /** \addtogroup Optimization Optimization and Regression
90 */
91 //@{
92 /** Solve Quadratic Programming Problem.
93
94 The quadraticProgramming() function implements the algorithm described in
95
96 D. Goldfarb, A. Idnani: <i>"A numerically stable dual method for solving
97 strictly convex quadratic programs"</i>, Mathematical Programming 27:1-33, 1983.
98
99 for the solution of (convex) quadratic programming problems by means of a primal-dual method.
100
101 <b>\#include</b> <vigra/quadprog.hxx> <br/>
102 Namespaces: vigra
103
104 <b>Declaration:</b>
105
106 \code
107 namespace vigra {
108 template <class T, class C1, class C2, class C3, class C4, class C5, class C6, class C7>
109 T
110 quadraticProgramming(MultiArrayView<2, T, C1> const & GG, MultiArrayView<2, T, C2> const & g,
111 MultiArrayView<2, T, C3> const & CE, MultiArrayView<2, T, C4> const & ce,
112 MultiArrayView<2, T, C5> const & CI, MultiArrayView<2, T, C6> const & ci,
113 MultiArrayView<2, T, C7> & x);
114 }
115 \endcode
116
117 The problem must be specified in the form:
118
119 \f{eqnarray*}
120 \mbox{minimize } &\,& \frac{1}{2} \mbox{\bf x}'\,\mbox{\bf G}\, \mbox{\bf x} + \mbox{\bf g}'\,\mbox{\bf x} \\
121 \mbox{subject to} &\,& \mbox{\bf C}_E\, \mbox{\bf x} = \mbox{\bf c}_e \\
122 &\,& \mbox{\bf C}_I\,\mbox{\bf x} \ge \mbox{\bf c}_i
123 \f}
124 Matrix <b>G</b> G must be symmetric positive definite, and matrix <b>C</b><sub>E</sub> must have full row rank.
125 Matrix and vector dimensions must be as follows:
126 <ul>
127 <li> <b>G</b>: [n * n], <b>g</b>: [n * 1]
128 <li> <b>C</b><sub>E</sub>: [me * n], <b>c</b><sub>e</sub>: [me * 1]
129 <li> <b>C</b><sub>I</sub>: [mi * n], <b>c</b><sub>i</sub>: [mi * 1]
130 <li> <b>x</b>: [n * 1]
131 </ul>
132
133 The function writes the optimal solution into the vector \a x and returns the cost of this solution.
134 If the problem is infeasible, <tt>std::numeric_limits::infinity()</tt> is returned. In this case
135 the value of vector \a x is undefined.
136
137 <b>Usage:</b>
138
139 Minimize <tt> f = 0.5 * x'*G*x + g'*x </tt> subject to <tt> -1 &lt;= x &lt;= 1</tt>.
140 The solution is <tt> x' = [1.0, 0.5, -1.0] </tt> with <tt> f = -22.625</tt>.
141 \code
142 double Gdata[] = {13.0, 12.0, -2.0,
143 12.0, 17.0, 6.0,
144 -2.0, 6.0, 12.0};
145
146 double gdata[] = {-22.0, -14.5, 13.0};
147
148 double CIdata[] = { 1.0, 0.0, 0.0,
149 0.0, 1.0, 0.0,
150 0.0, 0.0, 1.0,
151 -1.0, 0.0, 0.0,
152 0.0, -1.0, 0.0,
153 0.0, 0.0, -1.0};
154
155 double cidata[] = {-1.0, -1.0, -1.0, -1.0, -1.0, -1.0};
156
157 Matrix<double> G(3,3, Gdata),
158 g(3,1, gdata),
159 CE, // empty since there are no equality constraints
160 ce, // likewise
161 CI(7,3, CIdata),
162 ci(7,1, cidata),
163 x(3,1);
164
165 double f = quadraticProgramming(G, g, CE, ce, CI, ci, x);
166 \endcode
167
168 This algorithm can also be used to solve non-negative least squares problems
169 (see \ref nonnegativeLeastSquares() for an alternative algorithm). Consider the
170 problem to minimize <tt> f = (A*x - b)' * (A*x - b) </tt> subject to <tt> x &gt;= 0</tt>.
171 Expanding the product in the objective gives <tt>f = x'*A'*A*x - 2*b'*A*x + b'*b</tt>.
172 This is equivalent to the problem of minimizing <tt>fn = 0.5 * x'*G*x + g'*x</tt> with
173 <tt>G = A'*A</tt> and <tt>g = -A'*b</tt> (the constant term <tt>b'*b</tt> has no
174 effect on the optimal solution and can be dropped). The following code computes the
175 solution <tt>x' = [18.4493, 0, 4.50725]</tt>:
176 \code
177 double A_data[] = {
178 1, -3, 2,
179 -3, 10, -5,
180 2, -5, 6
181 };
182
183 double b_data[] = {
184 27,
185 -78,
186 64
187 };
188
189 Matrix<double> A(3, 3, A_data),
190 b(3, 1, b_data),
191 G = transpose(A)*A,
192 g = -(transpose(A)*b),
193 CE, // empty since there are no equality constraints
194 ce, // likewise
195 CI = identityMatrix<double>(3), // constrain all elements of x
196 ci(3, 1, 0.0), // ... to be non-negative
197 x(3, 1);
198
199 quadraticProgramming(G, g, CE, ce, CI, ci, x);
200 \endcode
201 */
202 doxygen_overloaded_function(template <...> unsigned int quadraticProgramming)
203
204 template <class T, class C1, class C2, class C3, class C4, class C5, class C6, class C7>
205 T
206 quadraticProgramming(MultiArrayView<2, T, C1> const & G, MultiArrayView<2, T, C2> const & g,
207 MultiArrayView<2, T, C3> const & CE, MultiArrayView<2, T, C4> const & ce,
208 MultiArrayView<2, T, C5> const & CI, MultiArrayView<2, T, C6> const & ci,
209 MultiArrayView<2, T, C7> & x)
210 {
211 using namespace linalg;
212 typedef typename MultiArrayShape<2>::type Shape;
213
214 int n = rowCount(g),
215 me = rowCount(ce),
216 mi = rowCount(ci),
217 constraintCount = me + mi;
218
219 vigra_precondition(columnCount(G) == n && rowCount(G) == n,
220 "quadraticProgramming(): Matrix shape mismatch between G and g.");
221 vigra_precondition(rowCount(x) == n,
222 "quadraticProgramming(): Output vector x has illegal shape.");
223 vigra_precondition((me > 0 && columnCount(CE) == n && rowCount(CE) == me) ||
224 (me == 0 && columnCount(CE) == 0),
225 "quadraticProgramming(): Matrix CE has illegal shape.");
226 vigra_precondition((mi > 0 && columnCount(CI) == n && rowCount(CI) == mi) ||
227 (mi == 0 && columnCount(CI) == 0),
228 "quadraticProgramming(): Matrix CI has illegal shape.");
229
230 Matrix<T> J = identityMatrix<T>(n);
231 {
232 Matrix<T> L(G.shape());
233 choleskyDecomposition(G, L);
234 // find unconstrained minimizer of the quadratic form 0.5 * x G x + g' x
235 choleskySolve(L, -g, x);
236 // compute the inverse of the factorized matrix G^-1, this is the initial value for J
237 linearSolveLowerTriangular(L, J, J);
238 }
239 // current solution value
240 T f_value = 0.5 * dot(g, x);
241
242 T epsilonZ = NumericTraits<T>::epsilon() * sq(J.norm(0)),
243 inf = std::numeric_limits<T>::infinity();
244
245 Matrix<T> R(n, n), r(constraintCount, 1), u(constraintCount,1);
246 T R_norm = NumericTraits<T>::one();
247
248 // incorporate equality constraints
249 for (int i=0; i < me; ++i)
250 {
251 MultiArrayView<2, T, C3> np = rowVector(CE, i);
252 Matrix<T> d = J*transpose(np);
253 Matrix<T> z = transpose(J).subarray(Shape(0, i), Shape(n,n))*subVector(d, i, n);
254 linearSolveUpperTriangular(R.subarray(Shape(0, 0), Shape(i,i)),
255 subVector(d, 0, i),
256 subVector(r, 0, i));
257 // compute step in primal space so that the constraint becomes satisfied
258 T step = (squaredNorm(z) <= epsilonZ) // i.e. z == 0
259 ? 0.0
260 : (-dot(np, x) + ce(i,0)) / dot(z, np);
261
262 x += step * z;
263 u(i,0) = step;
264 subVector(u, 0, i) -= step * subVector(r, 0, i);
265
266 f_value += 0.5 * sq(step) * dot(z, np);
267
268 vigra_precondition(vigra::detail::quadprogAddConstraint(R, J, d, i, R_norm),
269 "quadraticProgramming(): Equality constraints are linearly dependent.");
270 }
271 int activeConstraintCount = me;
272
273 // determine optimum solution and corresponding active inequality constraints
274 ArrayVector<int> activeSet(mi);
275 for (int i = 0; i < mi; ++i)
276 activeSet[i] = i;
277
278 int constraintToBeAdded = 0;
279 T ss = 0.0;
280 for (int i = activeConstraintCount-me; i < mi; ++i)
281 {
282 T s = dot(rowVector(CI, activeSet[i]), x) - ci(activeSet[i], 0);
283 if (s < ss)
284 {
285 ss = s;
286 constraintToBeAdded = i;
287 }
288 }
289
290 int iter = 0, maxIter = 10*mi;
291 while(iter++ < maxIter)
292 {
293 if (ss >= 0.0) // all constraints are satisfied
294 return f_value; // => solved!
295
296 // determine step direction in the primal space (through J, see the paper)
297 MultiArrayView<2, T, C5> np = rowVector(CI, activeSet[constraintToBeAdded]);
298 Matrix<T> d = J*transpose(np);
299 Matrix<T> z = transpose(J).subarray(Shape(0, activeConstraintCount), Shape(n,n))*subVector(d, activeConstraintCount, n);
300
301 // compute negative of the step direction in the dual space
302 linearSolveUpperTriangular(R.subarray(Shape(0, 0), Shape(activeConstraintCount,activeConstraintCount)),
303 subVector(d, 0, activeConstraintCount),
304 subVector(r, 0, activeConstraintCount));
305
306 // determine minimum step length in primal space such that activeSet[constraintToBeAdded] becomes feasible
307 T primalStep = (squaredNorm(z) <= epsilonZ) // i.e. z == 0
308 ? inf
309 : -ss / dot(z, np);
310
311 // determine maximum step length in dual space that doesn't violate dual feasibility
312 // and the corresponding index
313 T dualStep = inf;
314 int constraintToBeRemoved = 0;
315 for (int k = me; k < activeConstraintCount; ++k)
316 {
317 if (r(k,0) > 0.0)
318 {
319 if (u(k,0) / r(k,0) < dualStep)
320 {
321 dualStep = u(k,0) / r(k,0);
322 constraintToBeRemoved = k;
323 }
324 }
325 }
326
327 // the step is chosen as the minimum of dualStep and primalStep
328 T step = std::min(dualStep, primalStep);
329
330 // take step and update matrices
331
332 if (step == inf)
333 {
334 // case (i): no step in primal or dual space possible
335 return inf; // QPP is infeasible
336 }
337 if (primalStep == inf)
338 {
339 // case (ii): step in dual space
340 subVector(u, 0, activeConstraintCount) -= step * subVector(r, 0, activeConstraintCount);
341 vigra::detail::quadprogDeleteConstraint(R, J, u, activeConstraintCount, constraintToBeRemoved);
342 --activeConstraintCount;
343 std::swap(activeSet[constraintToBeRemoved-me], activeSet[activeConstraintCount-me]);
344 continue;
345 }
346
347 // case (iii): step in primal and dual space
348 x += step * z;
349 // update the solution value
350 f_value += 0.5 * sq(step) * dot(z, np);
351 // u = [u 1]' + step * [-r 1]
352 subVector(u, 0, activeConstraintCount) -= step * subVector(r, 0, activeConstraintCount);
353 u(activeConstraintCount,0) = step;
354
355 if (step == primalStep)
356 {
357 // add constraintToBeAdded to the active set
358 vigra::detail::quadprogAddConstraint(R, J, d, activeConstraintCount, R_norm);
359 std::swap(activeSet[constraintToBeAdded], activeSet[activeConstraintCount-me]);
360 ++activeConstraintCount;
361 }
362 else
363 {
364 // drop constraintToBeRemoved from the active set
365 vigra::detail::quadprogDeleteConstraint(R, J, u, activeConstraintCount, constraintToBeRemoved);
366 --activeConstraintCount;
367 std::swap(activeSet[constraintToBeRemoved-me], activeSet[activeConstraintCount-me]);
368 }
369
370 // update values of inactive inequality constraints
371 ss = 0.0;
372 for (int i = activeConstraintCount-me; i < mi; ++i)
373 {
374 // compute CI*x - ci with appropriate row permutation
375 T s = dot(rowVector(CI, activeSet[i]), x) - ci(activeSet[i], 0);
376 if (s < ss)
377 {
378 ss = s;
379 constraintToBeAdded = i;
380 }
381 }
382 }
383 return inf; // too many iterations
384 }
385
386 //@}
387
388 } // namespace vigra
389
390 #endif
vigra::linalg::columnVector
MultiArrayView< 2, T, C > columnVector(MultiArrayView< 2, T, C > const &m, MultiArrayIndex d)
Definition: matrix.hxx:727
vigra::linalg::rowCount
MultiArrayIndex rowCount(const MultiArrayView< 2, T, C > &x)
Definition: matrix.hxx:671
vigra::linalg::transpose
void transpose(const MultiArrayView< 2, T, C1 > &v, MultiArrayView< 2, T, C2 > &r)
Definition: matrix.hxx:965
vigra::linalg::linearSolveLowerTriangular
bool linearSolveLowerTriangular(const MultiArrayView< 2, T, C1 > &l, const MultiArrayView< 2, T, C2 > &b, MultiArrayView< 2, T, C3 > x)
Definition: linear_solve.hxx:1118
vigra::linalg::choleskySolve
void choleskySolve(MultiArrayView< 2, T, C1 > const &L, MultiArrayView< 2, T, C2 > const &b, MultiArrayView< 2, T, C3 > &x)
Definition: linear_solve.hxx:1168
vigra::linalg::sq
linalg::TemporaryMatrix< T > sq(MultiArrayView< 2, T, C > const &v)
vigra::doxygen_overloaded_function
doxygen_overloaded_function(template<...> void separableConvolveBlockwise) template< unsigned int N
Separated convolution on ChunkedArrays.
vigra::linalg::squaredNorm
Matrix< T, ALLLOC >::SquaredNormType squaredNorm(const Matrix< T, ALLLOC > &a)
vigra::MultiArrayShape< 2 >::type
TinyVector< MultiArrayIndex, N > type
Definition: multi_shape.hxx:272
vigra::linalg::subVector
MultiArrayView< 2, T, C > subVector(MultiArrayView< 2, T, C > const &m, int first, int end)
Definition: matrix.hxx:761
vigra::linalg::dot
NormTraits< T >::SquaredNormType dot(const MultiArrayView< 2, T, C1 > &x, const MultiArrayView< 2, T, C2 > &y)
Definition: matrix.hxx:1342
vigra::linalg::rowVector
MultiArrayView< 2, T, C > rowVector(MultiArrayView< 2, T, C > const &m, MultiArrayIndex d)
Definition: matrix.hxx:697
vigra::linalg::columnCount
MultiArrayIndex columnCount(const MultiArrayView< 2, T, C > &x)
Definition: matrix.hxx:684
vigra::abs
FFTWComplex< R >::NormType abs(const FFTWComplex< R > &a)
absolute value (= magnitude)
Definition: fftw3.hxx:1002
vigra::linalg::choleskyDecomposition
bool choleskyDecomposition(MultiArrayView< 2, T, C1 > const &A, MultiArrayView< 2, T, C2 > &L)
Definition: linear_solve.hxx:959
vigra::linalg::linearSolveUpperTriangular
bool linearSolveUpperTriangular(const MultiArrayView< 2, T, C1 > &r, const MultiArrayView< 2, T, C2 > &b, MultiArrayView< 2, T, C3 > x)
Definition: linear_solve.hxx:1068
vigra::quadraticProgramming
unsigned int quadraticProgramming(...)

© Ullrich Köthe (ullrich.koethe@iwr.uni-heidelberg.de) Heidelberg Collaboratory for Image Processing, University of Heidelberg, Germany | html generated using doxygen and Python vigra 1.11.1 (Fri May 19 2017)