File fix-issue-235.patch of Package lftp
From 764c5a3057357abdf6c272b0e67ff26c07e272f0 Mon Sep 17 00:00:00 2001
From: "Alexander V. Lukyanov" <lavv17f@gmail.com>
Date: Tue, 10 May 2016 17:09:37 +0300
Subject: [PATCH] Squashed patches for issue 235
---
src/FileCopy.cc | 6 ++--
src/FileSet.cc | 21 +++++------
src/FileSet.h | 10 ++++--
src/Http.cc | 107 +++++++++++++++++++++++++++++++++----------------------
src/Http.h | 1 +
src/MirrorJob.cc | 3 +-
src/NetAccess.cc | 87 +++++++++++++++++++++++++++++++++++++++++---
src/NetAccess.h | 7 ++++
src/xstring.h | 3 ++
9 files changed, 182 insertions(+), 63 deletions(-)
diff --git a/src/FileCopy.cc b/src/FileCopy.cc
index c71df2e..46de066 100644
--- a/src/FileCopy.cc
+++ b/src/FileCopy.cc
@@ -1238,8 +1238,10 @@ int FileCopyPeerFA::Get_LL(int len)
file.set(loc);
}
- size=NO_SIZE_YET;
- date=NO_DATE_YET;
+ if(want_size || size!=NO_SIZE)
+ WantSize();
+ if(want_date || date!=NO_DATE)
+ WantDate();
upload_state.Clear();
current->Timeout(0); // retry with new location.
diff --git a/src/FileSet.cc b/src/FileSet.cc
index 54afc89..2c7197e 100644
--- a/src/FileSet.cc
+++ b/src/FileSet.cc
@@ -50,10 +50,16 @@ void FileInfo::Merge(const FileInfo& f)
{
if(strcmp(basename_ptr(name),basename_ptr(f.name)))
return;
-// int sim=defined&f.defined;
- int dif=(~defined)&f.defined;
- if(dif&MODE)
+ MergeInfo(f,~defined);
+}
+void FileInfo::MergeInfo(const FileInfo& f,unsigned dif)
+{
+ dif&=f.defined;
+ if(dif&MODE) {
SetMode(f.mode);
+ if(mode!=SYMLINK && mode!=REDIRECT)
+ symlink.unset();
+ }
if(dif&DATE || (defined&DATE && f.defined&DATE && f.date.ts_prec<date.ts_prec))
SetDate(f.date,f.date.ts_prec);
if(dif&SIZE)
@@ -484,7 +490,7 @@ void FileSet::Count(int *d,int *f,int *s,int *o) const
if(f) (*f)++; break;
case(FileInfo::SYMLINK):
if(s) (*s)++; break;
- case(FileInfo::UNKNOWN):
+ default:
if(o) (*o)++;
}
}
@@ -720,12 +726,6 @@ FileInfo::FileInfo(const FileInfo &fi)
nlinks=fi.nlinks;
longname.set(fi.longname);
}
-FileInfo::FileInfo(const char *n)
-{
- Init();
- SetName(n);
-}
-
FileInfo::~FileInfo()
{
}
@@ -1004,6 +1004,7 @@ void FileInfo::MakeLongName()
case UNKNOWN: break;
case DIRECTORY: filetype_c='d'; break;
case SYMLINK: filetype_c='l'; break;
+ case REDIRECT: filetype_c='L'; break;
}
int mode1=(defined&MODE?mode:
(filetype_c=='d'?0755:(filetype_c=='l'?0777:0644)));
diff --git a/src/FileSet.h b/src/FileSet.h
index 6771fa9..7f456d0 100644
--- a/src/FileSet.h
+++ b/src/FileSet.h
@@ -51,6 +51,7 @@ public:
xstring name;
xstring longname;
xstring_c symlink;
+ xstring_c uri;
mode_t mode;
FileTimestamp date;
off_t size;
@@ -63,7 +64,8 @@ public:
UNKNOWN=0,
DIRECTORY,
SYMLINK,
- NORMAL
+ NORMAL,
+ REDIRECT,
};
type filetype;
@@ -85,7 +87,8 @@ public:
void Init();
FileInfo() { Init(); }
FileInfo(const FileInfo &fi);
- FileInfo(const char *n);
+ FileInfo(const char *n) { Init(); SetName(n); }
+ FileInfo(const xstring& n) { Init(); SetName(n); }
~FileInfo();
void SetName(const char *n) { name.set(n); def(NAME); }
@@ -100,10 +103,13 @@ public:
void SetDate(time_t t,int prec) { date.set(t,prec); def(DATE); }
void SetType(type t) { filetype=t; def(TYPE); }
void SetSymlink(const char *s) { symlink.set(s); filetype=SYMLINK; def(TYPE|SYMLINK_DEF); }
+ void SetRedirect(const char *s) { symlink.set(s); filetype=REDIRECT; def(TYPE|SYMLINK_DEF); }
+ const char *GetRedirect() const { return symlink; }
void SetSize(off_t s) { size=s; def(SIZE); }
void SetNlink(int n) { nlinks=n; def(NLINKS); }
void Merge(const FileInfo&);
+ void MergeInfo(const FileInfo& f,unsigned mask);
bool SameAs(const FileInfo *,int ignore) const;
bool OlderThan(time_t t) const;
diff --git a/src/Http.cc b/src/Http.cc
index 6591fc2..4ca83bd 100644
--- a/src/Http.cc
+++ b/src/Http.cc
@@ -222,6 +222,7 @@ void Http::MoveConnectionHere(Http *o)
void Http::DisconnectLL()
{
+ Enter(this);
rate_limit=0;
if(conn)
{
@@ -247,6 +248,7 @@ void Http::DisconnectLL()
last_url.unset();
ResetRequestData();
state=DISCONNECTED;
+ Leave(this);
}
void Http::ResetRequestData()
@@ -877,6 +879,10 @@ int Http::SendArrayInfoRequest()
name=&xstring::get_tmp(*name);
name->append('/');
}
+ if(fi->uri)
+ file_url.set(dir_file(GetConnectURL(),fi->uri));
+ else
+ file_url.unset();
SendRequest(array_send==fileset_for_info->count()-1 ? 0 : "keep-alive", *name);
req_count++;
}
@@ -1578,6 +1584,14 @@ int Http::Do()
}
goto pre_RECEIVING_BODY;
}
+ FileInfo *fi=fileset_for_info->curr();
+ if(H_REDIRECTED(status_code)) {
+ HandleRedirection();
+ if(location)
+ fi->SetRedirect(location);
+ } else if(H_2XX(status_code) && !fi->Has(fi->TYPE)) {
+ fi->SetType(last_uri.last_char()=='/'?fi->DIRECTORY:fi->NORMAL);
+ }
ProceedArrayInfo();
return MOVED;
}
@@ -1736,49 +1750,7 @@ int Http::Do()
if(H_REDIRECTED(status_code))
{
- bool is_url=(location && url::is_url(location));
- if(location && !is_url
- && mode==QUOTE_CMD && !strncasecmp(file,"POST ",5)
- && tunnel_state!=TUNNEL_WAITING)
- {
- const char *the_file=file;
-
- const char *scan=file+5;
- while(*scan==' ')
- scan++;
- char *the_post_file=alloca_strdup(scan);
- char *space=strchr(the_post_file,' ');
- if(space)
- *space=0;
- the_file=the_post_file;
-
- char *new_location=alloca_strdup2(GetConnectURL(),
- strlen(the_file)+strlen(location));
- int p_ind=url::path_index(new_location);
- if(location[0]=='/')
- strcpy(new_location+p_ind,location);
- else
- {
- if(the_file[0]=='/')
- strcpy(new_location+p_ind,the_file);
- else
- {
- char *slash=strrchr(new_location,'/');
- strcpy(slash+1,the_file);
- }
- char *slash=strrchr(new_location,'/');
- strcpy(slash+1,location);
- }
- location.set(new_location);
- } else if(is_url && !hftp) {
- ParsedURL url(location);
- if(url.proto.eq(GetProto()) && !xstrcasecmp(url.host,hostname)
- && user && !url.user) {
- // use the same user name after redirect to the same site.
- url.user.set(user);
- location.set_allocated(url.Combine());
- }
- }
+ HandleRedirection();
err.setf("%s (%s -> %s)",status+status_consumed,file.get(),
location?location.get():"nowhere");
code=FILE_MOVED;
@@ -1927,6 +1899,53 @@ system_error:
return MOVED;
}
+void Http::HandleRedirection()
+{
+ bool is_url=(location && url::is_url(location));
+ if(location && !is_url
+ && mode==QUOTE_CMD && !strncasecmp(file,"POST ",5)
+ && tunnel_state!=TUNNEL_WAITING)
+ {
+ const char *the_file=file;
+
+ const char *scan=file+5;
+ while(*scan==' ')
+ scan++;
+ char *the_post_file=alloca_strdup(scan);
+ char *space=strchr(the_post_file,' ');
+ if(space)
+ *space=0;
+ the_file=the_post_file;
+
+ char *new_location=alloca_strdup2(GetConnectURL(),
+ strlen(the_file)+strlen(location));
+ int p_ind=url::path_index(new_location);
+ if(location[0]=='/')
+ strcpy(new_location+p_ind,location);
+ else
+ {
+ if(the_file[0]=='/')
+ strcpy(new_location+p_ind,the_file);
+ else
+ {
+ char *slash=strrchr(new_location,'/');
+ strcpy(slash+1,the_file);
+ }
+ char *slash=strrchr(new_location,'/');
+ strcpy(slash+1,location);
+ }
+ location.set(new_location);
+ } else if(is_url && !hftp) {
+ ParsedURL url(location);
+ if(url.proto.eq(GetProto()) && !xstrcasecmp(url.host,hostname)
+ && user && !url.user) {
+ // use the same user name after redirect to the same site.
+ url.user.set(user);
+ location.set_allocated(url.Combine());
+ }
+ }
+}
+
FileAccess *Http::New() { return new Http(); }
FileAccess *HFtp::New() { return new HFtp(); }
@@ -1964,6 +1983,7 @@ int Http::Read(Buffer *buf,int size)
int res=DO_AGAIN;
if(state==RECEIVING_BODY && real_pos>=0)
{
+ Enter(this);
res=_Read(buf,size);
if(res>0)
{
@@ -1972,6 +1992,7 @@ int Http::Read(Buffer *buf,int size)
rate_limit->BytesGot(res);
TrySuccess();
}
+ Leave(this);
}
return res;
}
diff --git a/src/Http.h b/src/Http.h
index ad19912..6633f52 100644
--- a/src/Http.h
+++ b/src/Http.h
@@ -109,6 +109,7 @@ class Http : public NetAccess
int status_code;
void HandleHeaderLine(const char *name,const char *value);
static const xstring& extract_quoted_header_value(const char *value,const char **end=0);
+ void HandleRedirection();
void GetBetterConnection(int level);
void SetCookie(const char *val);
void MakeCookie(xstring &cookie,const char *host,const char *path);
diff --git a/src/MirrorJob.cc b/src/MirrorJob.cc
index 776e350..8697259 100644
--- a/src/MirrorJob.cc
+++ b/src/MirrorJob.cc
@@ -273,6 +273,7 @@ void MirrorJob::HandleFile(FileInfo *file)
switch(filetype)
{
case(FileInfo::NORMAL):
+ case(FileInfo::REDIRECT):
{
bool remove_target=false;
bool cont_this=false;
@@ -1215,7 +1216,7 @@ int MirrorJob::Do()
break;
// all jobs finished.
- if(remove_source_dirs) {
+ if(remove_source_dirs && source_dir.last_char()!='/') {
// remove source directory once.
remove_source_dirs=false;
if(script)
diff --git a/src/NetAccess.cc b/src/NetAccess.cc
index 18a9fdd..9d55473 100644
--- a/src/NetAccess.cc
+++ b/src/NetAccess.cc
@@ -404,6 +404,31 @@ do_again:
if(done)
return m;
+ if(redir_resolution) {
+ if(redir_session && redir_session->OpenMode()==FA::ARRAY_INFO) {
+ res=redir_session->Done();
+ if(res==FA::DO_AGAIN || res==FA::IN_PROGRESS)
+ return m;
+ redir_session->Close();
+ redir_fs->rewind();
+ FileInfo *fi=redir_fs->curr();
+ if(ResolveRedirect(fi))
+ return MOVED;
+ result->curr()->MergeInfo(*fi,~0U);
+ result->next();
+ }
+ redir_count=0;
+ for(FileInfo *fi=result->curr(); fi; fi=result->next()) {
+ if(ResolveRedirect(fi))
+ return MOVED;
+ }
+
+ FileAccess::cache->UpdateFileSet(session,"",FA::MP_LIST,result);
+ FileAccess::cache->UpdateFileSet(session,"",FA::LONG_LIST,result);
+ done=true;
+ return MOVED;
+ }
+
if(session->OpenMode()==FA::ARRAY_INFO)
{
res=session->Done();
@@ -412,10 +437,11 @@ do_again:
if(res==FA::IN_PROGRESS)
return m;
session->Close();
- FileAccess::cache->UpdateFileSet(session,"",FA::MP_LIST,result);
- FileAccess::cache->UpdateFileSet(session,"",FA::LONG_LIST,result);
- done=true;
- return MOVED;
+ // start redirection resolution.
+ redir_resolution=true;
+ result->rewind();
+ m=MOVED;
+ goto do_again;
}
if(!ubuf)
@@ -567,8 +593,59 @@ got_fileset:
return m;
}
+bool GenericParseListInfo::ResolveRedirect(const FileInfo *fi)
+{
+ if(fi->filetype!=fi->REDIRECT || redir_count>=max_redir)
+ return false;
+
+ redir_count++;
+ Log::global->Format(9,"ListInfo: resolving redirection %s -> %s\n",fi->name.get(),fi->GetRedirect());
+
+ Ref<FileInfo> redir_fi(new FileInfo());
+ redir_fi->Need(fi->need);
+
+ xstring loc(fi->GetRedirect());
+ ParsedURL u(loc,true);
+ if(!u.proto) {
+ // relative URI
+ redir_session=session->Clone();
+ if(loc[0]=='/' || fi->uri) {
+ if(loc[0]!='/') {
+ const char *slash=strrchr(fi->uri,'/');
+ if(slash)
+ loc.prepend(fi->uri,slash+1-fi->uri);
+ }
+ redir_fi->uri.set(loc);
+ redir_fi->name.set(loc);
+ redir_fi->name.url_decode();
+ } else {
+ loc.url_decode();
+ const char *slash=strrchr(fi->name,'/');
+ if(slash)
+ redir_fi->name.nset(fi->name,slash+1-fi->name);
+ redir_fi->name.append(loc);
+ }
+ } else { // u.proto
+ // absolute URL
+ redir_session=FileAccess::New(&u);
+ redir_fi->name.set(u.path?u.path.get():"/");
+ redir_fi->uri.set(url::path_ptr(u.orig_url));
+ }
+
+ if(!redir_fs)
+ redir_fs=new FileSet();
+ else
+ redir_fs->Empty();
+ redir_fs->Add(redir_fi.borrow());
+ redir_session->GetInfoArray(redir_fs.get_non_const());
+ redir_session->Roll();
+
+ return true;
+}
+
GenericParseListInfo::GenericParseListInfo(FileAccess *s,const char *p)
- : ListInfo(s,p)
+ : ListInfo(s,p), redir_resolution(false), redir_count(0),
+ max_redir(ResMgr::Query("xfer:max-redirections",0))
{
get_time_for_dirs=true;
can_get_prec_time=true;
diff --git a/src/NetAccess.h b/src/NetAccess.h
index 28e33ab..4972849 100644
--- a/src/NetAccess.h
+++ b/src/NetAccess.h
@@ -109,6 +109,13 @@ public:
class GenericParseListInfo : public ListInfo
{
+ bool redir_resolution;
+ int redir_count;
+ int max_redir;
+ FileAccessRef redir_session;
+ Ref<FileSet> redir_fs;
+ bool ResolveRedirect(const FileInfo *fi);
+
protected:
int mode;
SMTaskRef<IOBuffer> ubuf;
diff --git a/src/xstring.h b/src/xstring.h
index 8bba5db..55918ba 100644
--- a/src/xstring.h
+++ b/src/xstring.h
@@ -145,6 +145,7 @@ public:
bool eq_nc(const char *s) const { return !xstrcasecmp(buf,s); }
size_t length() const { return xstrlen(buf); }
void set_length(size_t n) { if(buf) buf[n]=0; }
+ char last_char() const { size_t len=length(); return len>0?buf[len-1]:0; }
void unset() { xfree(buf); buf=0; }
void _set(const char *s) { buf=xstrdup(s); }
@@ -205,6 +206,8 @@ public:
xstring& set_substr(int start,size_t sublen,const char *,size_t);
xstring& set_substr(int start,size_t sublen,const char *);
xstring& set_substr(int start,size_t sublen,const xstring &s) { return set_substr(start,sublen,s.get(),s.length()); }
+ xstring& prepend(const char *s,size_t len) { return set_substr(0,0,s,len); }
+ xstring& prepend(const xstring &s) { return prepend(s.get(),s.length()); }
xstring& append(const char *s);
xstring& append(char c);
--
2.8.2
