Revision 6efdad6f2163928f8cdf415801700dc9ef5995ad authored by Henrique B.R on 24 September 2021, 16:18:04 UTC, committed by Henrique B.R on 24 September 2021, 16:18:04 UTC
1 parent a822c48
Test_hw_multigrid_mixed_48_rb.cc
/*************************************************************************************
Grid physics library, www.github.com/paboyle/Grid
Source file: ./tests/Test_dwf_hdcr.cc
Copyright (C) 2015
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
See the full license in the file "LICENSE" in the top level distribution directory
*************************************************************************************/
/* END LEGAL */
#include <Grid/Grid.h>
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidual.h>
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
#include <Grid/algorithms/iterative/BiCGSTAB.h>
using namespace std;
using namespace Grid;
// TODO
//
// Coarse Grid axpby_ssp_pminus // Inherit from spProj5pm
// Coarse Grid axpby_ssp_pplus
template<class Field,class Coeff_t>
class CayleyBase : public SparseMatrixBase<Field>
{
public:
int Ls;
// protected:
RealD mass;
RealD M5;
// Save arguments to SetCoefficientsInternal
Vector<Coeff_t> _gamma;
RealD _zolo_hi;
RealD _b;
RealD _c;
// Cayley form Moebius (tanh and zolotarev)
Vector<Coeff_t> omega;
Vector<Coeff_t> bs; // S dependent coeffs
Vector<Coeff_t> cs;
Vector<Coeff_t> as;
// For preconditioning Cayley form
Vector<Coeff_t> bee;
Vector<Coeff_t> cee;
Vector<Coeff_t> aee;
Vector<Coeff_t> beo;
Vector<Coeff_t> ceo;
Vector<Coeff_t> aeo;
// LDU factorisation of the eeoo matrix
Vector<Coeff_t> lee;
Vector<Coeff_t> leem;
Vector<Coeff_t> uee;
Vector<Coeff_t> ueem;
Vector<Coeff_t> dee;
public:
CayleyBase(RealD _M5, RealD _mass, int _Ls, RealD b_, RealD c_) :
M5(_M5),
mass(_mass),
Ls(_Ls),
_b(b_),
_c(c_)
{
RealD eps = 1.0;
Approx::zolotarev_data *zdata = Approx::higham(eps,this->Ls);// eps is ignored for higham
this->SetCoefficientsTanh(zdata,1.0,0.0);
Approx::zolotarev_free(zdata);
}
/////////////////////////////////////////////////////////
// Replicates functionality
// Use a common base class approach
/////////////////////////////////////////////////////////
// Tanh
void SetCoefficientsTanh(Approx::zolotarev_data *zdata,RealD b,RealD c)
{
Vector<Coeff_t> gamma(this->Ls);
for(int s=0;s<this->Ls;s++) gamma[s] = zdata->gamma[s];
SetCoefficientsInternal(1.0,gamma,b,c);
}
//Zolo
void SetCoefficientsZolotarev(RealD zolo_hi,Approx::zolotarev_data *zdata,RealD b,RealD c)
{
Vector<Coeff_t> gamma(this->Ls);
for(int s=0;s<this->Ls;s++) gamma[s] = zdata->gamma[s];
SetCoefficientsInternal(zolo_hi,gamma,b,c);
}
//Zolo
void SetCoefficientsInternal(RealD zolo_hi,Vector<Coeff_t> & gamma,RealD b,RealD c)
{
int Ls=this->Ls;
///////////////////////////////////////////////////////////
// The Cayley coeffs (unprec)
///////////////////////////////////////////////////////////
assert(gamma.size()==Ls);
omega.resize(Ls);
bs.resize(Ls);
cs.resize(Ls);
as.resize(Ls);
double bpc = b+c;
double bmc = b-c;
_b = b;
_c = c;
_gamma = gamma; // Save the parameters so we can change mass later.
_zolo_hi= zolo_hi;
for(int i=0; i < Ls; i++){
as[i] = 1.0;
omega[i] = _gamma[i]*_zolo_hi; //NB reciprocal relative to Chroma NEF code
assert(omega[i]!=Coeff_t(0.0));
bs[i] = 0.5*(bpc/omega[i] + bmc);
cs[i] = 0.5*(bpc/omega[i] - bmc);
}
////////////////////////////////////////////////////////
// Constants for the preconditioned matrix Cayley form
////////////////////////////////////////////////////////
bee.resize(Ls);
cee.resize(Ls);
beo.resize(Ls);
ceo.resize(Ls);
for(int i=0;i<Ls;i++){
bee[i]=as[i]*(bs[i]*(4.0-this->M5) +1.0);
assert(bee[i]!=Coeff_t(0.0));
cee[i]=as[i]*(1.0-cs[i]*(4.0-this->M5));
beo[i]=as[i]*bs[i];
ceo[i]=-as[i]*cs[i];
}
aee.resize(Ls);
aeo.resize(Ls);
for(int i=0;i<Ls;i++){
aee[i]=cee[i];
aeo[i]=ceo[i];
}
//////////////////////////////////////////
// LDU decomposition of eeoo
//////////////////////////////////////////
dee.resize(Ls);
lee.resize(Ls);
leem.resize(Ls);
uee.resize(Ls);
ueem.resize(Ls);
for(int i=0;i<Ls;i++){
dee[i] = bee[i];
if ( i < Ls-1 ) {
assert(bee[i]!=Coeff_t(0.0));
assert(bee[0]!=Coeff_t(0.0));
lee[i] =-cee[i+1]/bee[i]; // sub-diag entry on the ith column
leem[i]=mass*cee[Ls-1]/bee[0];
for(int j=0;j<i;j++) {
assert(bee[j+1]!=Coeff_t(0.0));
leem[i]*= aee[j]/bee[j+1];
}
uee[i] =-aee[i]/bee[i]; // up-diag entry on the ith row
ueem[i]=mass;
for(int j=1;j<=i;j++) ueem[i]*= cee[j]/bee[j];
ueem[i]*= aee[0]/bee[0];
} else {
lee[i] =0.0;
leem[i]=0.0;
uee[i] =0.0;
ueem[i]=0.0;
}
}
{
Coeff_t delta_d=mass*cee[Ls-1];
for(int j=0;j<Ls-1;j++) {
assert(bee[j] != Coeff_t(0.0));
delta_d *= cee[j]/bee[j];
}
dee[Ls-1] += delta_d;
}
};
//////////////////////////////
// M and Mdag
//////////////////////////////
virtual void Mdiag (const Field &in, Field &out) {assert(0);}
virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);};
virtual void MdirAll (const Field &in, std::vector<Field> &out){assert(0);};
virtual void DW (const Field &psi, Field &chi)=0;
virtual void DWDag (const Field &psi, Field &chi)=0;
void M (const Field &psi, Field &chi)
{
Field Din(psi.Grid());
Meooe5D(psi,Din);
DW(Din,chi);
axpby(chi,1.0,1.0,chi,psi);
M5D(psi,chi);
}
void Mdag (const Field &psi, Field &chi)
{
Field Din(psi.Grid());
DWDag(psi,Din);
MeooeDag5D(Din,chi);
M5Ddag(psi,chi);
axpby (chi,1.0,1.0,chi,psi);
}
/////////////////////////////////
// P and Pdag - might be needed
/////////////////////////////////
void P(const Field &psi, Field &chi)
{
int Ls= this->Ls;
chi=Zero();
for(int s=0;s<Ls;s++){
axpby_ssp_pminus(chi,1.0,chi,1.0,psi,s,s);
axpby_ssp_pplus (chi,1.0,chi,1.0,psi,s,(s+1)%Ls);
}
}
void Pdag(const Field &psi, Field &chi)
{
int Ls= this->Ls;
chi=Zero();
for(int s=0;s<Ls;s++){
axpby_ssp_pminus(chi,1.0,chi,1.0,psi,s,s);
axpby_ssp_pplus (chi,1.0,chi,1.0,psi,s,(s-1+Ls)%Ls);
}
}
////////////////////////////////////////////////////////
// Depends: Dw, M5D, M5Ddag, Meooe5D, MeooeDag5D,
////////////////////////////////////////////////////////
void M5D (const Field &psi, Field &chi)
{
int Ls=this->Ls;
Vector<Coeff_t> diag (Ls,1.0);
Vector<Coeff_t> upper(Ls,-1.0); upper[Ls-1]=mass;
Vector<Coeff_t> lower(Ls,-1.0); lower[0] =mass;
M5D(psi,chi,chi,lower,diag,upper);
}
void M5Ddag (const Field &psi, Field &chi)
{
int Ls=this->Ls;
Vector<Coeff_t> diag(Ls,1.0);
Vector<Coeff_t> upper(Ls,-1.0);
Vector<Coeff_t> lower(Ls,-1.0);
upper[Ls-1]=-mass*upper[Ls-1];
lower[0] =-mass*lower[0];
M5Ddag(psi,chi,chi,lower,diag,upper);
}
void Meooe5D (const Field &psi, Field &Din)
{
int Ls=this->Ls;
Vector<Coeff_t> diag = bs;
Vector<Coeff_t> upper= cs;
Vector<Coeff_t> lower= cs;
upper[Ls-1]=-mass*upper[Ls-1];
lower[0] =-mass*lower[0];
M5D(psi,psi,Din,lower,diag,upper);
}
void MeooeDag5D (const Field &psi, Field &Din)
{
int Ls=this->Ls;
Vector<Coeff_t> diag =bs;
Vector<Coeff_t> upper=cs;
Vector<Coeff_t> lower=cs;
for (int s=0;s<Ls;s++){
if ( s== 0 ) {
upper[s] = cs[s+1];
lower[s] =-mass*cs[Ls-1];
} else if ( s==(Ls-1) ) {
upper[s] =-mass*cs[0];
lower[s] = cs[s-1];
} else {
upper[s] = cs[s+1];
lower[s] = cs[s-1];
}
upper[s] = conjugate(upper[s]);
lower[s] = conjugate(lower[s]);
diag[s] = conjugate(diag[s]);
}
M5Ddag(psi,psi,Din,lower,diag,upper);
}
void M5D(const Field &psi_i,
const Field &phi_i,
Field &chi_i,
Vector<Coeff_t> &lower,
Vector<Coeff_t> &diag,
Vector<Coeff_t> &upper)
{
chi_i.Checkerboard()=psi_i.Checkerboard();
GridBase *grid=psi_i.Grid();
autoView(psi , psi_i,AcceleratorRead);
autoView(phi , phi_i,AcceleratorRead);
autoView(chi , chi_i,AcceleratorWrite);
assert(phi.Checkerboard() == psi.Checkerboard());
auto pdiag = &diag[0];
auto pupper = &upper[0];
auto plower = &lower[0];
int Ls =this->Ls;
// 10 = 3 complex mult + 2 complex add
// Flops = 10.0*(Nc*Ns) *Ls*vol (/2 for red black counting)
uint64_t nloop = grid->oSites()/Ls;
const int Nsimd = Field::vector_type::Nsimd();
accelerator_for(sss,nloop,Nsimd,{
uint64_t ss= sss*Ls;
typedef decltype(coalescedRead(psi[0])) spinor;
spinor tmp1, tmp2;
for(int s=0;s<Ls;s++){
uint64_t idx_u = ss+((s+1)%Ls);
uint64_t idx_l = ss+((s+Ls-1)%Ls);
spProj5m(tmp1,psi(idx_u)); // Need routines for this
spProj5p(tmp2,psi(idx_l));
coalescedWrite(chi[ss+s],pdiag[s]*phi(ss+s)+pupper[s]*tmp1+plower[s]*tmp2);
}
});
}
void M5Ddag(const Field &psi_i,
const Field &phi_i,
Field &chi_i,
Vector<Coeff_t> &lower,
Vector<Coeff_t> &diag,
Vector<Coeff_t> &upper)
{
chi_i.Checkerboard()=psi_i.Checkerboard();
GridBase *grid=psi_i.Grid();
autoView(psi , psi_i,AcceleratorRead);
autoView(phi , phi_i,AcceleratorRead);
autoView(chi , chi_i,AcceleratorWrite);
assert(phi.Checkerboard() == psi.Checkerboard());
auto pdiag = &diag[0];
auto pupper = &upper[0];
auto plower = &lower[0];
int Ls=this->Ls;
uint64_t nloop = grid->oSites()/Ls;
const int Nsimd = Field::vector_type::Nsimd();
accelerator_for(sss,nloop,Nsimd,{
uint64_t ss=sss*Ls;
typedef decltype(coalescedRead(psi[0])) spinor;
spinor tmp1,tmp2;
for(int s=0;s<Ls;s++){
uint64_t idx_u = ss+((s+1)%Ls);
uint64_t idx_l = ss+((s+Ls-1)%Ls);
spProj5p(tmp1,psi(idx_u));
spProj5m(tmp2,psi(idx_l));
coalescedWrite(chi[ss+s],pdiag[s]*phi(ss+s)+pupper[s]*tmp1+plower[s]*tmp2);
}
});
}
};
template<class Fobj,class CComplex,int nbasis>
class CoarseCayleyFermion : public CayleyBase< Lattice<iVector<CComplex,nbasis > > , ComplexD >
{
public:
typedef iVector<CComplex,nbasis > siteVector;
typedef Lattice<CComplex > CoarseComplexField;
typedef Lattice<siteVector> CoarseVector;
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
typedef iMatrix<CComplex,nbasis > Cobj;
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
typedef Lattice<Fobj > FineField;
// Similar to the CoarseOperator but add 5D support.
Geometry geom;
GridBase *Coarse5D;
GridBase *Coarse4D;
CartesianStencil<siteVector,siteVector,int> Stencil;
CoarsenedMatrix<Fobj,CComplex,nbasis> &Dw;
GridBase * Grid(void) { return Coarse5D; }; // this is all the linalg routines need to know
CoarseCayleyFermion(GridCartesian &CoarseGrid4,
GridCartesian &CoarseGrid5,
CoarsenedMatrix<Fobj,CComplex,nbasis> &_Dw,
RealD M5, RealD mass, int Ls, RealD b, RealD c) :
CayleyBase<CoarseVector,ComplexD>(M5,mass,Ls,b,c),
Coarse4D(&CoarseGrid4),
Coarse5D(&CoarseGrid5),
Dw(_Dw),
geom(CoarseGrid5._ndimension),
Stencil( &CoarseGrid5,geom.npoint,Even,geom.directions,geom.displacements,0)
{
};
public:
void Project( CoarseVector &C )
{
const int Nsimd = CComplex::Nsimd();
autoView(Cv,C, AcceleratorWrite);
int Ls = this->Ls;
for(int s=0;s<Ls;s++){
accelerator_for(sU, Coarse4D->oSites(), Nsimd, {
int sF= sU*Ls+s;
auto tmp = coalescedRead(Cv[sF]);
coalescedWrite(Cv[sF],tmp);
});
}
}
////////////////////////////////////////////////
// This is specific to Coarse Grid Cayley
////////////////////////////////////////////////
virtual void Mdiag (const CoarseVector &in, CoarseVector &out)
{
std::vector<CoarseVector> allout(9,in.Grid());
this->MdirAll(in,allout);
out = allout[8];
}
virtual void Mdir (const CoarseVector &in, CoarseVector &out,int dir, int disp)
{
assert(0);
}
virtual void MdirAll (const CoarseVector &in, std::vector<CoarseVector> &out)
{
conformable(Coarse5D,in.Grid());
SimpleCompressor<siteVector> compressor;
Stencil.HaloExchange(in,compressor);
typedef LatticeView<Cobj> Aview;
const int Nsimd = CComplex::Nsimd();
// Ls loop for2D
int Ls=this->Ls;
siteVector *CBp=Stencil.CommBuf();
int ptype;
int nb2=nbasis/2;
autoView(in_v , in, AcceleratorRead);
autoView(st, Stencil, AcceleratorRead);
for(int point=0;point<geom.npoint;point++){
autoView(out_v, out[point], AcceleratorWrite);
autoView(Aview,Dw.A[point],AcceleratorRead);
accelerator_for2d(sF, Coarse5D->oSites(), b, nbasis, Nsimd, {
typedef decltype(coalescedRead(in_v[0])) calcVector;
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
int sU = sF/Ls;
int s = sF%Ls;
calcComplex res = Zero();
calcVector nbr;
int ptype;
StencilEntry *SE=st.GetEntry(ptype,point,sF);
if(SE->_is_local) {
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
} else {
nbr = coalescedRead(CBp[SE->_offset]);
}
acceleratorSynchronise();
for(int bb=0;bb<nbasis;bb++) {
res = res + coalescedRead(Aview[sU](b,bb))*nbr(bb);
}
coalescedWrite(out_v[sF](b),res);
});
}
}
void DW (const CoarseVector &in, CoarseVector &out)
{
conformable(Coarse5D,in.Grid());
conformable(in.Grid(),out.Grid());
SimpleCompressor<siteVector> compressor;
Stencil.HaloExchange(in,compressor);
typedef LatticeView<Cobj> Aview;
const int Nsimd = CComplex::Nsimd();
// Ls loop for2D
int Ls=this->Ls;
Vector<Aview> AcceleratorViewContainer;
for(int p=0;p<geom.npoint;p++) AcceleratorViewContainer.push_back(Dw.A[p].View(AcceleratorRead));
Aview *Aview_p = & AcceleratorViewContainer[0];
autoView(in_v , in, AcceleratorRead);
autoView(out_v, out, AcceleratorWrite);
autoView(st, Stencil, AcceleratorRead);
siteVector *CBp=Stencil.CommBuf();
int ptype;
int nb2=nbasis/2;
accelerator_for2d(sF, Coarse5D->oSites(), b, nbasis, Nsimd, {
typedef decltype(coalescedRead(in_v[0])) calcVector;
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
int sU = sF/Ls;
int s = sF%Ls;
calcComplex res = Zero();
{
calcVector nbr;
int ptype;
for(int point=0;point<geom.npoint;point++){
StencilEntry *SE=st.GetEntry(ptype,point,sF);
if(SE->_is_local) {
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
} else {
nbr = coalescedRead(CBp[SE->_offset]);
}
acceleratorSynchronise();
for(int bb=0;bb<nbasis;bb++) {
res = res + coalescedRead(Aview_p[point][sU](b,bb))*nbr(bb);
}
}
}
coalescedWrite(out_v[sF](b),res);
});
for(int p=0;p<geom.npoint;p++) AcceleratorViewContainer[p].ViewClose();
};
void DWDag (const CoarseVector &in, CoarseVector &out)
{
// Inefficient G5 hermitian use
CoarseVector tmp(Grid());
G5C(tmp, in); //There has to be a better way
DW(tmp, out);
G5C(out, out);
};
typedef Aggregation<Fobj,CComplex,nbasis> Aggregates;
void PromoteFromSubspace(Aggregates &_Aggregates,CoarseVector &C,FineField &F)
{
auto FineGrid4 = _Aggregates.FineGrid;
FineField F4(FineGrid4);
CoarseVector C4(Coarse4D);
for(int s=0;s<this->Ls;s++){
ExtractSlice(C4,C,s,0);
_Aggregates.PromoteFromSubspace(C4,F4);
InsertSlice(F4,F,s,0);
}
}
void ProjectToSubspace(Aggregates &_Aggregates,CoarseVector &C,FineField &F)
{
auto FineGrid4 = _Aggregates.FineGrid;
FineField F4(FineGrid4);
CoarseVector C4(Coarse4D);
for(int s=0;s<this->Ls;s++){
ExtractSlice(F4,F,s,0);
_Aggregates.ProjectToSubspace (C4,F4);
InsertSlice(C4,C,s,0);
}
Project(C);
}
template<class Ddwf>
void Test(Aggregates &_Aggregates,GridBase *FineGrid, Ddwf &_Ddwf)
{
typedef Lattice<Fobj> FineField;
CoarseVector Cin(Coarse5D);
CoarseVector Cout(Coarse5D);
CoarseVector CFout(Coarse5D);
FineField Fin(FineGrid);
FineField Fout(FineGrid);
std::vector<int> seeds({1,2,3,4,5});
GridParallelRNG RNG(Coarse5D); RNG.SeedFixedIntegers(seeds);
gaussian(RNG,Cin);
PromoteFromSubspace(_Aggregates,Cin,Fin);
ProjectToSubspace(_Aggregates,Cin,Fin);
std::cout << GridLogMessage<< "************ "<<std::endl;
std::cout << GridLogMessage<< " Testing M "<<std::endl;
std::cout << GridLogMessage<< "************ "<<std::endl;
// Coarse operator
this->M(Cin,Cout);
this->Project(Cout);
std::cout << GridLogMessage<< " Cout "<<norm2(Cout)<<std::endl;
// Fine projected operator
PromoteFromSubspace(_Aggregates,Cin,Fin);
_Ddwf.M(Fin,Fout);
ProjectToSubspace(_Aggregates,CFout,Fout);
std::cout << GridLogMessage<< " CFout "<<norm2(CFout)<<std::endl;
CFout = CFout-Cout;
std::cout << GridLogMessage<< " diff "<<norm2(CFout)<<std::endl;
std::cout << GridLogMessage<< "************ "<<std::endl;
std::cout << GridLogMessage<< " Testing Mdag "<<std::endl;
std::cout << GridLogMessage<< "************ "<<std::endl;
// Coarse operator
this->Mdag(Cin,Cout);
this->Project(Cout);
std::cout << GridLogMessage<< " Cout "<<norm2(Cout)<<std::endl;
// Fine operator
_Ddwf.Mdag(Fin,Fout);
ProjectToSubspace(_Aggregates,CFout,Fout);
std::cout << GridLogMessage<< " CFout "<<norm2(CFout)<<std::endl;
CFout = CFout-Cout;
std::cout << GridLogMessage<< " diff "<<norm2(CFout)<<std::endl;
}
virtual std::vector<int> Directions(void) { return geom.directions;};
virtual std::vector<int> Displacements(void){ return geom.displacements;};
};
template<class Field> class SchurSolverWrapper : public LinearFunction<Field> {
private:
CheckerBoardedSparseMatrixBase<Field> & _Matrix;
SchurRedBlackBase<Field> & _Solver;
public:
/////////////////////////////////////////////////////
// Wrap the usual normal equations trick
/////////////////////////////////////////////////////
SchurSolverWrapper(CheckerBoardedSparseMatrixBase<Field> &Matrix,
SchurRedBlackBase<Field> &Solver)
: _Matrix(Matrix), _Solver(Solver) {};
void operator() (const Field &in, Field &out){
_Solver(_Matrix,in,out); // Mdag M out = Mdag in
}
};
template<class Field> class SolverWrapper : public LinearFunction<Field> {
private:
LinearOperatorBase<Field> & _Matrix;
OperatorFunction<Field> & _Solver;
LinearFunction<Field> & _Guess;
public:
/////////////////////////////////////////////////////
// Wrap the usual normal equations trick
/////////////////////////////////////////////////////
SolverWrapper(LinearOperatorBase<Field> &Matrix,
OperatorFunction<Field> &Solver,
LinearFunction<Field> &Guess)
: _Matrix(Matrix), _Solver(Solver), _Guess(Guess) {};
void operator() (const Field &in, Field &out){
_Guess(in,out);
_Solver(_Matrix,in,out); // Mdag M out = Mdag in
}
};
// Must use a non-hermitian solver
template<class Matrix,class Field>
class PVdagMLinearOperator : public LinearOperatorBase<Field> {
Matrix &_Mat;
Matrix &_PV;
public:
PVdagMLinearOperator(Matrix &Mat,Matrix &PV): _Mat(Mat),_PV(PV){};
virtual std::vector<int> Directions(void) { return _Mat.Directions();};
virtual std::vector<int> Displacements(void){ return _Mat.Displacements();};
void OpDiag (const Field &in, Field &out) {
assert(0);
}
void OpDir (const Field &in, Field &out,int dir,int disp) {
assert(0);
}
void OpDirAll (const Field &in, std::vector<Field> &out){
assert(0);
};
void Op (const Field &in, Field &out){
Field tmp(in.Grid());
_Mat.M(in,tmp);
_PV.Mdag(tmp,out);
}
void AdjOp (const Field &in, Field &out){
Field tmp(in.Grid());
_PV.M(tmp,out);
_Mat.Mdag(in,tmp);
}
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
assert(0);
}
void HermOp(const Field &in, Field &out){
assert(0);
}
};
RealD InverseApproximation(RealD x){
return 1.0/x;
}
template<class Field,class Matrix> class ChebyshevSmoother : public LinearFunction<Field>
{
public:
typedef LinearOperatorBase<Field> FineOperator;
Matrix & _SmootherMatrix;
FineOperator & _SmootherOperator;
Chebyshev<Field> Cheby;
ChebyshevSmoother(RealD _lo,RealD _hi,int _ord, FineOperator &SmootherOperator,Matrix &SmootherMatrix) :
_SmootherOperator(SmootherOperator),
_SmootherMatrix(SmootherMatrix),
Cheby(_lo,_hi,_ord,InverseApproximation)
{};
void operator() (const Field &in, Field &out)
{
Field tmp(in.Grid());
MdagMLinearOperator<Matrix,Field> MdagMOp(_SmootherMatrix);
_SmootherOperator.AdjOp(in,tmp);
Cheby(MdagMOp,tmp,out);
}
};
template<class Fobj,class CComplex,int nbasis, class CoarseSolver>
class MGPreconditioner : public LinearFunction< Lattice<Fobj> > {
public:
typedef Aggregation<Fobj,CComplex,nbasis> Aggregates;
typedef typename Aggregation<Fobj,CComplex,nbasis>::CoarseVector CoarseVector;
typedef typename Aggregation<Fobj,CComplex,nbasis>::CoarseMatrix CoarseMatrix;
typedef typename Aggregation<Fobj,CComplex,nbasis>::FineField FineField;
typedef LinearOperatorBase<FineField> FineOperator;
typedef LinearFunction <FineField> FineSmoother;
typedef CoarseCayleyFermion<Fobj,CComplex,nbasis> CoarseOperator;
// typedef SparseMatrixBase<CoarseVector> CoarseOperator;
Aggregates & _Aggregates;
FineOperator & _FineOperator;
FineSmoother & _PreSmoother;
FineSmoother & _PostSmoother;
CoarseOperator & _CoarseOperator;
CoarseSolver & _CoarseSolve;
int level; void Level(int lv) {level = lv; };
MGPreconditioner(Aggregates &Agg,
FineOperator &Fine,
FineSmoother &PreSmoother,
FineSmoother &PostSmoother,
CoarseOperator &CoarseOperator_,
CoarseSolver &CoarseSolve_)
: _Aggregates(Agg),
_FineOperator(Fine),
_PreSmoother(PreSmoother),
_PostSmoother(PostSmoother),
_CoarseOperator(CoarseOperator_),
_CoarseSolve(CoarseSolve_),
level(1) { }
virtual void operator()(const FineField &in, FineField & out)
{
auto CoarseGrid = _CoarseOperator.Grid();
CoarseVector Csrc(CoarseGrid);
CoarseVector Csol(CoarseGrid);
FineField vec1(in.Grid());
FineField vec2(in.Grid());
std::cout<<GridLogMessage << "Calling PreSmoother " <<std::endl;
// std::cout<<GridLogMessage << "Calling PreSmoother input residual "<<norm2(in) <<std::endl;
double t;
// Fine Smoother
t=-usecond();
_PreSmoother(in,out);
t+=usecond();
std::cout<<GridLogMessage << "PreSmoother took "<< t/1000.0<< "ms" <<std::endl;
// Update the residual
_FineOperator.Op(out,vec1); sub(vec1, in ,vec1);
// std::cout<<GridLogMessage <<"Residual-1 now " <<norm2(vec1)<<std::endl;
// Fine to Coarse
t=-usecond();
_CoarseOperator.ProjectToSubspace(_Aggregates,Csrc,vec1);
// _Aggregates.ProjectToSubspace (Csrc,vec1);
t+=usecond();
std::cout<<GridLogMessage << "Project to coarse took "<< t/1000.0<< "ms" <<std::endl;
// Coarse correction
t=-usecond();
_CoarseSolve(Csrc,Csol);
//Csol=Zero();
t+=usecond();
std::cout<<GridLogMessage << "Coarse solve took "<< t/1000.0<< "ms" <<std::endl;
// Coarse to Fine
t=-usecond();
_CoarseOperator.PromoteFromSubspace(_Aggregates,Csol,vec1);
// _Aggregates.PromoteFromSubspace(Csol,vec1);
add(out,out,vec1);
t+=usecond();
std::cout<<GridLogMessage << "Promote to this level took "<< t/1000.0<< "ms" <<std::endl;
// Residual
_FineOperator.Op(out,vec1); sub(vec1 ,in , vec1);
// std::cout<<GridLogMessage <<"Residual-2 now " <<norm2(vec1)<<std::endl;
// Fine Smoother
t=-usecond();
_PostSmoother(vec1,vec2);
t+=usecond();
std::cout<<GridLogMessage << "PostSmoother took "<< t/1000.0<< "ms" <<std::endl;
add( out,out,vec2);
std::cout<<GridLogMessage << "Done " <<std::endl;
}
};
template<class Fobj,class CComplex,int nbasis, class CoarseSolver>
class HDCRPreconditioner : public LinearFunction< Lattice<Fobj> > {
public:
typedef Aggregation<Fobj,CComplex,nbasis> Aggregates;
typedef typename Aggregation<Fobj,CComplex,nbasis>::CoarseVector CoarseVector;
typedef typename Aggregation<Fobj,CComplex,nbasis>::CoarseMatrix CoarseMatrix;
typedef typename Aggregation<Fobj,CComplex,nbasis>::FineField FineField;
typedef LinearOperatorBase<FineField> FineOperator;
typedef LinearFunction <FineField> FineSmoother;
//typedef CoarseCayleyFermion<Fobj,CComplex,nbasis> CoarseOperator;
typedef SparseMatrixBase<CoarseVector> CoarseOperator;
Aggregates & _Aggregates;
FineOperator & _FineOperator;
FineSmoother & _PreSmoother;
FineSmoother & _PostSmoother;
CoarseOperator & _CoarseOperator;
CoarseSolver & _CoarseSolve;
int level; void Level(int lv) {level = lv; };
HDCRPreconditioner(Aggregates &Agg,
FineOperator &Fine,
FineSmoother &PreSmoother,
FineSmoother &PostSmoother,
CoarseOperator &CoarseOperator_,
CoarseSolver &CoarseSolve_)
: _Aggregates(Agg),
_FineOperator(Fine),
_PreSmoother(PreSmoother),
_PostSmoother(PostSmoother),
_CoarseOperator(CoarseOperator_),
_CoarseSolve(CoarseSolve_),
level(1) { }
virtual void operator()(const FineField &in, FineField & out)
{
auto CoarseGrid = _CoarseOperator.Grid();
CoarseVector Csrc(CoarseGrid);
CoarseVector g5Csrc(CoarseGrid);
CoarseVector Csol(CoarseGrid);
FineField vec1(in.Grid());
FineField vec2(in.Grid());
std::cout<<GridLogMessage<<"\t\t\t" << "Calling PreSmoother " <<std::endl;
double t;
// Fine Smoother
t=-usecond();
_PreSmoother(in,out);
t+=usecond();
std::cout<<GridLogMessage<<"\t\t\t" << "PreSmoother took "<< t/1000.0<< "ms" <<std::endl;
// Update the residual
_FineOperator.Op(out,vec1); sub(vec1, in ,vec1);
// Fine to Coarse
// Based on a coarsening of G5R5 D
// Solves Ddwf out = in
// Coarse operator is g5R5 Ddwf solves g5R5 Ddwf out = in
t=-usecond();
G5R5(vec2,vec1);
_Aggregates.ProjectToSubspace (Csrc,vec2);
t+=usecond();
std::cout<<GridLogMessage<<"\t\t\t" << "Project to coarse took "<< t/1000.0<< "ms" <<std::endl;
// Coarse correction
t=-usecond();
_CoarseSolve(Csrc,Csol);
t+=usecond();
std::cout<<GridLogMessage<<"\t\t\t" << "Coarse solve took "<< t/1000.0<< "ms" <<std::endl;
// Coarse to Fine
t=-usecond();
_Aggregates.PromoteFromSubspace(Csol,vec1);
add(out,out,vec1);
t+=usecond();
std::cout<<GridLogMessage<<"\t\t\t" << "Promote to this level took "<< t/1000.0<< "ms" <<std::endl;
// Residual
_FineOperator.Op(out,vec1); sub(vec1 ,in , vec1);
// Fine Smoother
t=-usecond();
_PostSmoother(vec1,vec2);
t+=usecond();
std::cout<<GridLogMessage<<"\t\t\t" << "PostSmoother took "<< t/1000.0<< "ms" <<std::endl;
add( out,out,vec2);
}
};
int main (int argc, char ** argv)
{
Grid_init(&argc,&argv);
const int Ls=24;
GridCartesian * UGrid = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
GridCartesian * FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
///////////////////////////////////////////////////
// Construct a coarsened grid; utility for this?
///////////////////////////////////////////////////
std::vector<int> block ({2,2,2,2}); // 4,2,2,2 gets worse
std::vector<int> blockc ({1,1,1,1});
const int nbasis= 24;
const int nbasisc= 40; // decrease, not improvement
auto clatt = GridDefaultLatt();
for(int d=0;d<clatt.size();d++){
clatt[d] = clatt[d]/block[d];
}
auto cclatt = clatt;
for(int d=0;d<clatt.size();d++){
cclatt[d] = clatt[d]/blockc[d];
}
GridCartesian *Coarse4d = SpaceTimeGrid::makeFourDimGrid(clatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());;
GridCartesian *Coarse5d = SpaceTimeGrid::makeFiveDimGrid(Ls,Coarse4d);
// GridRedBlackCartesian * Coarse4dRB = SpaceTimeGrid::makeFourDimRedBlackGrid(Coarse4d);
// GridRedBlackCartesian * Coarse5dRB = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,Coarse4d);
GridCartesian *CoarseCoarse4d = SpaceTimeGrid::makeFourDimGrid(cclatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());;
GridCartesian *CoarseCoarse5d = SpaceTimeGrid::makeFiveDimGrid(1,CoarseCoarse4d);
// GridRedBlackCartesian * CoarseCoarse4dRB = SpaceTimeGrid::makeFourDimRedBlackGrid(CoarseCoarse4d);
GridRedBlackCartesian * CoarseCoarse5dRB = SpaceTimeGrid::makeFiveDimRedBlackGrid(1,CoarseCoarse4d);
std::vector<int> seeds({1,2,3,4});
GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds);
GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds);
GridParallelRNG CRNG(Coarse4d);CRNG.SeedFixedIntegers(seeds);
LatticeGaugeField Umu(UGrid);
#if 0
SU3::TepidConfiguration(RNG4,Umu);
RealD M5=1.0;
#else
std::string file("./ckpoint_lat.1000");
FieldMetaData header;
NerscIO::readConfiguration(Umu,header,file);
RealD M5=1.8;
#endif
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << "Building g5R5 hermitian DWF operator" <<std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
RealD mass=0.00078;
WilsonFermionR Dw(Umu,*UGrid,*UrbGrid,-M5);
DomainWallFermionR Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
DomainWallFermionR Dpv (Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0,M5);
typedef Aggregation<vSpinColourVector,vTComplex,nbasis> Subspace;
typedef CoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> CoarseOperator;
typedef CoarseOperator::CoarseVector CoarseVector;
typedef CoarseOperator::siteVector siteVector;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << "Calling Aggregation class to build subspace" <<std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
// How to find criticall mass?
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.75); // 600 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.80); // 800 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.82); // 1023 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.85); // 1428 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.87); // 1900 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.90); // 3900 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.92); // 6200 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.94); // 8882 iters
WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.95); // 9170 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.96); // 8882 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.97); // 8406 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-0.99); // 6900 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-1.01); // 6397 iters
// WilsonFermionR Dw_null(Umu,*UGrid,*UrbGrid,-1.00); // 5900 iters
MdagMLinearOperator<WilsonFermionR,LatticeFermion> MdagM_Dw(Dw_null);
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << "Testing Wilson criticality " <<std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
/*
ConjugateGradient<LatticeFermion> WilsonCG(1.0e-10,40000);
LatticeFermion w_src(UGrid); w_src=1.0;
LatticeFermion w_res(UGrid);
WilsonCG(MdagM_Dw,w_src,w_res);
exit(0);
*/
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << " 4D subspace build " <<std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
Subspace Aggregates4D(Coarse4d,UGrid,0);
assert ( (nbasis & 0x1)==0);
int nb=nbasis/2;
Gamma g5(Gamma::Algebra::Gamma5);
Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,3.0,300,150,150,0.0); // now at 26 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,3.0,300,150,150,0.0); // now at 26 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,3.0,500,150,150,0.0); // now at 26 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,500,150,150,0.0); // now at 26 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,500,150,150,0.0); //35
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,600,100,250,0.0); //39
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,600,250,100,0.0); //39
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,600,100,100,0.0); //36
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,600,250,100,0.0); //39
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,600,250,50,0.0);// 39
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,600,250,250,0.0);// 35
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,250,250,250,0.0);// 38 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,50.0,4.0,250,250,100,0.0);// 40 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,50.0,4.0,500,100,100,0.0);// 38 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,4.0,500,100,100,0.0);// 36 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,3.0,500,100,100,0.0);// 37 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,1.0,1000,400,400,0.0);// 38 iter
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,0.5,1000,400,400,0.0);// 39 iter HDCR smooth 14
// Aggregates4D.CreateSubspaceChebyshev(RNG4,MdagM_Dw,nb,60.0,0.1,1000,400,400,0.0);// 41
for(int n=0;n<nb;n++){
Aggregates4D.subspace[nbasis-1-n]= Aggregates4D.subspace[n] - g5 * Aggregates4D.subspace[n];
Aggregates4D.subspace[n] = Aggregates4D.subspace[n] + g5 * Aggregates4D.subspace[n];
}
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << " Coarsen the Dw operator " <<std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
typedef CoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> Level1Op4;
typedef CoarseCayleyFermion<vSpinColourVector,vTComplex,nbasis> Level1Op5;
Level1Op4 c_Dw (*Coarse4d,0);
NonHermitianLinearOperator<WilsonFermionR,LatticeFermion> LinOpDw(Dw);
c_Dw.CoarsenOperator(UGrid,LinOpDw,Aggregates4D); // contains the M5 from Dw(-M5)
// c_Dw.Test(Aggregates4D,UGrid,LinOpDw);
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << " Build coarse DWF operator " <<std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
Level1Op5 c_Dwf (*Coarse4d,*Coarse5d,c_Dw,M5, mass, Ls, 1.0,0.0);
// c_Dwf.Test(Aggregates4D,FGrid,Ddwf);
MdagMLinearOperator<Level1Op5,CoarseVector> MdagM_cDwf(c_Dwf);
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << "Build 5D coarse deflation space" << std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
int nbc=nbasisc/2;
typedef CoarsenedMatrix<siteVector,iScalar<vTComplex>,nbasisc> Level2Op;
typedef Aggregation<siteVector,iScalar<vTComplex>,nbasisc> CoarseSubspace;
CoarseSubspace CoarseAggregates(CoarseCoarse5d,Coarse5d,0);
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << "Build Chebyshev space in coarse operator "<< std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
// CoarseAggregates.CreateSubspaceChebyshev(CRNG,MdagM_cDwf,nbc,40.0,0.01,300,150,100,0.0);
CoarseAggregates.CreateSubspaceChebyshev(CRNG,MdagM_cDwf,nbc,40.0,0.005,500,150,100,0.0);
{
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << "Applying G5R5 projection of coarse operator "<< std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
CoarseVector A(Coarse5d), B(Coarse5d);
for(int n=0;n<nbc;n++){
G5R5(B,CoarseAggregates.subspace[n]);
A = CoarseAggregates.subspace[n];
CoarseAggregates.subspace[n] = A+B; // 1+G5R5 // eigen value of G5R5 is +1
CoarseAggregates.subspace[n+nbc]= A-B; // 1-G5R5 // eigen value of G5R5 is -1
}
}
Gamma5R5HermitianLinearOperator<Level1Op5,CoarseVector> L1Hdwf(c_Dwf);
Level2Op cc_Dwf (*CoarseCoarse5d,*CoarseCoarse5dRB,1); // say it is hermitian
cc_Dwf.CoarsenOperator(Coarse5d,L1Hdwf,CoarseAggregates);
// cc_Dwf.Test(CoarseAggregates,Coarse5d,L1Hdwf);
typedef Level2Op::CoarseVector CoarseCoarseVector;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << " Testing fine and coarse solvers " <<std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
RealD tol=1.0e-8;
int MaxIt = 10000;
CoarseVector c_src(Coarse5d); c_src=1.0;
CoarseVector c_res(Coarse5d);
LatticeFermion f_src(FGrid); f_src=1.0;
LatticeFermion f_res(FGrid);
LatticeFermion f_src_e(FrbGrid); f_src_e=1.0;
LatticeFermion f_res_e(FrbGrid);
CoarseCoarseVector cc_src(CoarseCoarse5d); cc_src=1.0;
ConjugateGradient<CoarseVector> CoarseCG(tol,MaxIt);
ConjugateGradient<LatticeFermion> FineCG(tol,MaxIt);
NonHermitianLinearOperator<DomainWallFermionR,LatticeFermion> FineM(Ddwf);
MdagMLinearOperator<DomainWallFermionR,LatticeFermion> FineMdagM(Ddwf); // M^\dag M
NonHermitianLinearOperator<Level1Op5,CoarseVector> CoarseM(c_Dwf);
MdagMLinearOperator<Level1Op5,CoarseVector> CoarseMdagM(c_Dwf);
NonHermitianLinearOperator<Level2Op,CoarseCoarseVector> CoarseCoarseM(cc_Dwf);
MdagMLinearOperator<Level2Op,CoarseCoarseVector> CoarseCoarseMdagM(cc_Dwf);
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << " Fine Hw PowerMethod "<< std::endl;
LatticeFermion w_src(UGrid);
w_src=1.0;
PowerMethod<LatticeFermion> PM; PM(MdagM_Dw,w_src);
std::cout<<GridLogMessage << " Coarse PowerMethod "<< std::endl;
c_src=1.0;
PowerMethod<CoarseVector> cPM; cPM(CoarseMdagM,c_src);
cc_src=1.0;
PowerMethod<CoarseCoarseVector> ccPM; ccPM(CoarseCoarseMdagM,cc_src);
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << " Running CoarseCoarse grid Lanczos "<< std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
// 37s, 26 iter
int cNk=128;
int cNm=256;
int cNstop=128;
RealD IRL_lo=0.01;
RealD IRL_hi=16.0;
int IRL_ord=201;
/*
// int cNk=100; -- slower 27 iters
int cNk=128; //-- 26 iters, but slower
int cNm=192;
int cNstop=128;
RealD IRL_lo=0.005;
RealD IRL_hi=10.0;
int IRL_ord=101;
*/
MdagMLinearOperator<Level2Op,CoarseCoarseVector> IRLHermOpL2(cc_Dwf);
Chebyshev<CoarseCoarseVector> IRLChebyL2(IRL_lo,IRL_hi,IRL_ord);
FunctionHermOp<CoarseCoarseVector> IRLOpChebyL2(IRLChebyL2,IRLHermOpL2);
PlainHermOp<CoarseCoarseVector> IRLOpL2 (IRLHermOpL2);
ImplicitlyRestartedLanczos<CoarseCoarseVector> IRLL2(IRLOpChebyL2,IRLOpL2,cNstop,cNk,cNm,1.0e-3,20);
int cNconv;
cNm=0;
std::vector<RealD> eval2(cNm);
std::vector<CoarseCoarseVector> evec2(cNm,CoarseCoarse5d);
cc_src=1.0;
// IRLL2.calc(eval2,evec2,cc_src,cNconv);
std::vector<RealD> tols ({0.005,0.001});
std::vector<RealD> c_los ({0.1,0.05});
std::vector<RealD> c_his ({22.0});
std::vector<RealD> f_los ({0.5,0.2});
std::vector<RealD> f_his ({60.0});
std::vector<int> ws ({2,3});
std::vector<int> c_ords ({32,24});
std::vector<int> f_ords ({20,16});
for(auto w : ws ) {
for(auto tol : tols ) {
for(auto f_ord : f_ords ) {
for(auto c_ord : c_ords ) {
for(auto c_lo : c_los ) {
for(auto c_hi : c_his ) {
for(auto f_lo : f_los ) {
for(auto f_hi : f_his ) {
ZeroGuesser<CoarseVector> CoarseZeroGuesser;
ZeroGuesser<CoarseCoarseVector> CoarseCoarseZeroGuesser;
ConjugateGradient<CoarseCoarseVector> CoarseCoarseCG(tol,10000);
ZeroGuesser<CoarseCoarseVector> CoarseCoarseGuesser;
SchurRedBlackDiagMooeeSolve<CoarseCoarseVector> CoarseCoarseRBCG(CoarseCoarseCG);
SchurSolverWrapper<CoarseCoarseVector> CoarseCoarseSolver(cc_Dwf,CoarseCoarseRBCG);
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << "Building 3 level hdcr "<< std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
// NormalEquations<CoarseCoarseVector> CoarseCoarseCGNE(cc_Dwf,CoarseCoarseCG,CoarseCoarseZeroGuesser);
{
typedef HDCRPreconditioner<siteVector,iScalar<vTComplex>,nbasisc,LinearFunction<CoarseCoarseVector> > CoarseMG;
typedef MGPreconditioner<vSpinColourVector, vTComplex,nbasis, LinearFunction<CoarseVector> > ThreeLevelMG;
// MultiGrid preconditioner acting on the coarse space <-> coarsecoarse space
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother1(0.5,22.0,c_ord,CoarseM,c_Dwf); // 37s, 26 iter
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother2(0.5,22.0,c_ord,CoarseM,c_Dwf);
ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother(c_lo,c_hi,c_ord,CoarseM,c_Dwf); // 37s, 26 iter
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother1(0.5,22.0,7,CoarseM,c_Dwf); // 38s, 26 iter
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother2(0.5,22.0,7,CoarseM,c_Dwf);
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother1(0.4,22.0,7,CoarseM,c_Dwf); // 41s, 27 iter
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother2(0.4,22.0,7,CoarseM,c_Dwf);
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother1(0.6,22.0,6,CoarseM,c_Dwf); // 26 iter
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother2(0.6,22.0,6,CoarseM,c_Dwf);
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother1(0.5,22.0,5,CoarseM,c_Dwf); // 33 iter, 55s
// ChebyshevSmoother<CoarseVector, Level1Op5 > CoarseSmoother2(0.5,22.0,5,CoarseM,c_Dwf);
CoarseMG Level2Precon (CoarseAggregates,
CoarseM,
CoarseSmoother,
CoarseSmoother,
cc_Dwf,
CoarseCoarseSolver);
Level2Precon.Level(2);
//PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> L2PGCR(0.5, 100, CoarseM,Level2Precon,16,16); // 26 iter, 37s
// PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> L2PGCR(0.0, 1, CoarseM,Level2Precon,2,2); // 296 s, 50 iter
// PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> L2PGCR(0.0, 1, CoarseM,Level2Precon,2,2); // 250 s, 37 iter
PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> L2PGCR(0.0, 1, CoarseM,Level2Precon,2,2);
//PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> L2PGCR(1.0, 100, CoarseM,Level2Precon,16,16); // 35 iter, 45s
//PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> L2PGCR(0.6, 100, CoarseM,Level2Precon,16,16); // 26,38 (diifferene is measurement noise)
//PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> L2PGCR(0.2, 100, CoarseM,Level2Precon,16,16); // 26 iter, 47s
L2PGCR.Level(2);
// Wrap the 2nd level solver in a MultiGrid preconditioner acting on the fine space
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.5,60.0,14,FineM,Ddwf); // 26 iter, 39s
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.5,60.0,14,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.5,60.0,12,FineM,Ddwf); // 25 iter, 38s
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.5,60.0,16,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.5,60.0,12,FineM,Ddwf); // 23 iter, 39s
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.5,60.0,20,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.5,60.0,10,FineM,Ddwf);24 iter, 44s
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.5,60.0,24,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.5,60.0,12,FineM,Ddwf); // odd convergence tail at 10^-9 ish
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.1,60.0,24,FineM,Ddwf); // 33 iter, waas O(10-9 by 26)
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.5,60.0,12,FineM,Ddwf); // 25 iter, 39s
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.5,60.0,18,FineM,Ddwf); //
ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother(f_lo,f_hi,f_ord,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.5,60.0,11,FineM,Ddwf); // 33 iter, 49s
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.5,60.0,11,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.5,60.0,12,FineM,Ddwf); // 26 iter, 37s
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.5,60.0,12,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.4,60.0,12,FineM,Ddwf); // iter 26 no change in final residual
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.4,60.0,12,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.3,60.0,12,FineM,Ddwf); // 27 iter 39s.
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.3,60.0,12,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(0.3,60.0,13,FineM,Ddwf); // 26 iter, but slower
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(0.3,60.0,13,FineM,Ddwf);
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother1(1.0,60.0,12,FineM,Ddwf); // 34 iter, slower
// ChebyshevSmoother<LatticeFermion,DomainWallFermionR> FineSmoother2(1.0,60.0,12,FineM,Ddwf);
ThreeLevelMG ThreeLevelPrecon(Aggregates4D,
FineM,
FineSmoother,
FineSmoother,
c_Dwf,
L2PGCR);
ThreeLevelPrecon.Level(1);
PrecGeneralisedConjugateResidualNonHermitian<LatticeFermion> L1PGCR(1.0e-8,1000,FineM,ThreeLevelPrecon,16,16);
L1PGCR.Level(1);
f_res=Zero();
L1PGCR(f_src,f_res);
}
}}}}
}}}
}
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
std::cout<<GridLogMessage << "Done "<< std::endl;
std::cout<<GridLogMessage << "**************************************************"<< std::endl;
Grid_finalize();
}
Computing file changes ...
