基于邻域嵌入的超分辨率方法理解与实现
<link rel="stylesheet" href="https://csdnimg.cn/release/blogv2/dist/mdeditor/css/editerView/ck_htmledit_views-b5506197d8.css">
<div id="content_views" class="markdown_views">
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
<path stroke-linecap="round" d="M5,0 0,2.5 5,5z" id="raphael-marker-block" style="-webkit-tap-highlight-color: rgba(0, 0, 0, 0);"></path>
</svg>
<h2 id="声明"><a name="t0"></a><a name="t0"></a>声明:</h2>
本文为博主原创,只是个人理解,如有理解错误的地方,欢迎指正。代码部分有参考导师提供的部分代码,来源不详。
- 1
参考文献:
Super-Resolution through Neighbor Embedding
(http://doi.ieeecomputersociety.org/10.1109/CVPR.2004.243)
- 1
- 2
算法描述:
给定一副低分辨率图像(LR)和多幅训练图像(SR training images),通过流型学习,得到目标高分辨率图像(SR)。
详细步骤:
- 降质
对训练图像降质(比如目标SR是LR的N倍大,则把训练图像缩小N倍)得到LR training images; - 分块
把LR和LR training images分成大小s*s ,重叠区域宽为overlap的重叠图像块(image pitch),SR training images则是Ns*Ns大的重叠图像块(image pitch),其中low-resolution training images中的每一块对应SR training images中每一块; - 特征提取
计算LR和LR training images各像素点的一阶梯度和二阶梯度,用一个特征向量代表一个图像块,特征向量的集合分别为XT,XS。对于SR training images,只需要每个pitch内每个像素减去块内所有像素值的平均值,这里为什么减去均值的原因是,超分辨率的本质是要找回由于图像降质过程丢失的高频细节信息,高频细节信息是由低分辨率图像中的中频信息所决定,而不是整个低分辨率图像。减去它的均值其实就是中频信息。SR training pitch同样是用一个向量表示,集合记为YS; - NE
对于XT中的每一个列向量,在XS中找到K个与之最相似(欧式距离最近)的列向量(neighbor),求出每个neighbor的权值,然后除以所有权值之和,使这K个权值相加为一(归一化);
前面提到每个LR training pitch对应一个SR training pitch,即用前面选出的K个neighbor对应的SR training pitch分别乘以对应的权重,加起来即为重建的image pitch的特征向量。
重复步骤4,直到所有的XT中向量都处理完,此时得到一个重叠的image pitch集合,记为YT; - 图像重建
现在已知YT,只要把YT中的特征向量恢复到图像即可。只需按顺序把YT中的每个向量代表的图像块恢复到原来的位置,重叠区域值为相邻image pitch在该区域的平均值表示,最后加上LR每个图像块的中频信息,即SR每个图像块像素值减去平均值,即可求出目标SR。
代码实现 :
MATLAB语言:
%——————————————————————————————————————————————————————————
%主程序
function example
s = 3; %3*3的图像块
overlap = 2; %重叠区域宽为2
K = 5; %在XS中查找K个neighbor
for i = 1:X %X幅训练图像
RGB_YS = imread(’.\pic…bmp’,‘bmp’);%输入训练图像的地址
RGB_XS = downsize(RGB_YS,4); %假设放大4倍
YIQ = rgb2ntsc(RGB_XS);XS = YIQ(:,:,1);
%这里是把RGB图像转换为YIQ图像,然后针对Y放大,具体转换原因文献里
%有提到,这里就不赘述了。
YIQ = rgb2ntsc(RGB_YS);YS = YIQ(:,:,1);
[XSp,XSrow,XScol] = overlapcut(XS, s, overlap);
[YSp,YSrow,YScol] = overlapcut(YS, 4*s, 4*overlap);
XSv2 = findgradient(XSp,XS,XSrow,XScol);
%计算每个像素值的梯度值
YSv2 = findfeature(YSp,YSrow,YScol);
%计算YS的特征向量
if i==1
XSv = XSv2;
YSv = YSv2;
else
XSv(:,size(XSv,2)+1:size(XSv,2)+size(XSv2,2)) = XSv2;
YSv(:,size(YSv,2)+1:size(YSv,2)+size(YSv2,2)) = YSv2;
%添加到XS,YS集合中
end
end
RGB_XT = imread(’.\pic\low.bmp’,‘bmp’);
YIQ = rgb2ntsc(RGB_XT); XT = YIQ(:,:,1);
YIQ = rgb2ntsc(upsize(RGB_XT,4));IQ_YT = YIQ(:,:,2:3);
%save IQ,为了后面重建的时候恢复成rgb图像
[XTp,XTrow,XTcol] = overlapcut(XT, 3, 2);
XTmean = findmean(XTp,XTrow,XTcol);%提取中频信息
XTv = findgradient(XTp,XT,XTrow,XTcol);
YTv = naneighbor(XTv,XSv,YSv,K);%计算出YT的特征向量集合
ET = findimage1(YTv,XTrow,XTcol,XTmean,IQ_YT,12,8); %重建
RGB_ET = ntsc2rgb(ET);%YIQ恢复到RGB图像
figure;image(RGB_XT);axis off;title(‘input low-res’);
figure;image(RGB_ET);axis off;title(‘output super-res’);
%——————————————————————————————————————————————————————————
%降质
function B = downsize(A,ratio)
[m,n,~] = size(A);
newm = floor(m/ratio);
newn = floor(n/ratio);
A = double(A);
B = zeros(newm,newn,3);
for i=1:ratio:m
minx = min(i+ratio-1,m);
for j=1:ratio:n
miny = min(j+ratio-1,n);
temp = round(sum(sum(A(i:minx,j:miny,:),2))/((minx-i+1)*(miny-j+1)));
ii = (i-1)/ratio+1;
jj=(j-1)/ratio+1;
B(ii,jj,:) = temp;
if miny == n
break;
end
end
if minx == m
break;
end
end
B = uint8(B);
%——————————————————————————————————————————————————————————
%分块
function [B,row,col] = overlapcut(A,s,overlap)
[m,n] = size(A);
row = ceil((m-overlap)/(s-overlap));
col = ceil((n-overlap)/(s-overlap));
B = zeros(s,s,rowcol);
for i=1:row
for j=1:col
index = (i-1)col + j;
ii = min((i-1)(s-overlap)+s,m)-(i-1)(s-overlap);
jj = min((j-1)(s-overlap)+s,n)-(j-1)(s-overlap);
B(1:ii,1:jj,index) = A((i-1)(s-overlap)+1:min((i-1)(s-overlap)+s,m),(j-1)(s-overlap)+1:min((j-1)(s-overlap)+s,n));
end
end
%——————————————————————————————————————————————————————————
%SR training image特征提取
function f = findfeature(A,row,col)
[~,s,n] = size(A); %n is # of patches
if (rowcol~=n)
fprintf(‘input parameter error!’);
return;
end
f = zeros(ss,n);
for i=1:n
temp = A(:,:,i);
vector = reshape(temp’,ss,1);
vector = vector - mean(vector);
f(:,i) = vector; %all piexl values (-mean) in the patches
end
%——————————————————————————————————————————————————————————
%LR和LR training image 的梯度值
function f = findgradient(A,B,row,col)
component of some image
[~,s,n] = size(A); %n is # of patches
[ii,jj] = size(B);
if (rowcol~=n)
fprintf(‘input parameter error!’);
return;
end
grad = zeros(ii,jj,4);
for i=1:ii %for each pixel in B matrix
for j=1:jj
if (j+1<=jj) && (j-1>=1)
grad(i,j,1) = B(i,j+1)-B(i,j-1);
end
if (i+1<=ii) && (i-1>=1)
grad(i,j,2) = B(i+1,j)-B(i-1,j);
end
if (i+2<=ii) && (i-2>=1)
grad(i,j,3) = B(i+2,j)-2B(i,j)+B(i-2,j);
end
if (j+2<=jj) && (j-2>=1)
grad(i,j,4) = B(i,j+2)-2B(i,j)+B(i,j-2);
end
end
end
f = zeros(4ss,n);
for p=1:n
i = ceil(p/col); % left top position of p-th patch
j = mod(p-1,col)+1; % in grad matrix is (i,j)
add = 0;
for si=0????-1
for sj=0????-1
for num=1:size(grad,3)
add = add+1;
f(add,p) = grad(i+si,j+sj,num);
end
end
end
end
%——————————————————————————————————————————————————————————
%NE算法
function [YT,U,neighborhood] = naneighbor(XT,XS,YS,K)
[~,T] = size(XT);
[M,~] = size(YS);
neighbors<span class=“hljs-transposed_variable”>n’,K);
neighborhood = zeros(K,T);
for i = 1:T
temp = XT(:,i);
distance = dist2(temp’,XS’);
[~,index] = sort(distance’);
neighborhood(:,i) = index(2:(K+1));
end
tol=1e-4; % regularlizer in case constrained fits are ill conditioned
U = zeros(K,T);
for ii=1:T
z = XS(:,neighborhood(:,ii))-repmat(XT(:,ii),1,K); % shift ith pt to origin
C = z’z; % local covariance
if trace©==0
C = C + eye(K,K)tol; % regularlization
else
C = C + eye(K,K)toltrace©;
end
U(:,ii) = C<span class=“hljs-built_in”>ones(K,1); % solve Cu=1
U(:,ii) = U(:,ii)/sum(U(:,ii)); % enforce sum(u)=1
end;
YT = zeros(M,T);
for ii = 1:T
YT(:,ii) = YS(:,neighborhood(:,ii))U(:,ii);
end
%——————————————————————————————————————————————————————————
%提取中频信息
function XTmean = findmean(XTp,row,col)
[,,n] = size(XTp);
if (n~=rowcol)
fprintf(‘input parameter error!’)
return;
end
XTmean = zeros(row,col);
for i=1:row
for j=1:col
index = (i-1)col+j;
XTmean(i,j) = mean(mean(XTp(:,:,index)));
end
end
%——————————————————————————————————————————————————————————
%重建图像
function YT = findimage1(YTv,prow,pcol,XTmean,IQ,s,overlap)
[row,col,~] = size(IQ);
[~,N] = size(YTv);
if (N~=prowpcol)
fprintf(‘input parameter error!’);
return;
end
Y = zeros(prow(s-overlap)+overlap,pcol(s-overlap)+overlap);
flag = zeros(prow(s-overlap)+overlap,pcol*(s-overlap)+overlap);
for i=1:prow
for j=1:pcol
for t=1????
Y((s-overlap)(i-1)+t,(s-overlap)(j-1)+1:(s-overlap)(j-1)+s) = XTmean(i,j) + Y((s-overlap)(i-1)+t,(s-overlap)(j-1)+1:(s-overlap)(j-1)+s)+ YTv(s*(t-1)+1????t,(i-1)pcol+j)’;
flag((s-overlap)(i-1)+t,(s-overlap)(j-1)+1:(s-overlap)(j-1)+s)= flag((s-overlap)(i-1)+t,(s-overlap)(j-1)+1:(s-overlap)(j-1)+s) +1;
end
end
end
Y = Y(1:row,1:col);
flag = flag(1:row,1:col);
YT = zeros(row,col,3);
YT(:,:,1) = Y./flag;
YT(:,:,[2,3]) = IQ(:,:????;
fprintf(‘Done.<span class=“hljs-transposed_variable”>n’);
%——————————————————————————————————————————————————————————
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
- 113
- 114
- 115
- 116
- 117
- 118
- 119
- 120
- 121
- 122
- 123
- 124
- 125
- 126
- 127
- 128
- 129
- 130
- 131
- 132
- 133
- 134
- 135
- 136
- 137
- 138
- 139
- 140
- 141
- 142
- 143
- 144
- 145
- 146
- 147
- 148
- 149
- 150
- 151
- 152
- 153
- 154
- 155
- 156
- 157
- 158
- 159
- 160
- 161
- 162
- 163
- 164
- 165
- 166
- 167
- 168
- 169
- 170
- 171
- 172
- 173
- 174
- 175
- 176
- 177
- 178
- 179
- 180
- 181
- 182
- 183
- 184
- 185
- 186
- 187
- 188
- 189
- 190
- 191
- 192
- 193
- 194
- 195
- 196
- 197
- 198
- 199
- 200
- 201
- 202
- 203
- 204
- 205
- 206
- 207
- 208
- 209
- 210
- 211
推荐阅读