util.cc 58 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/* <!-- copyright */
/*
 * aria2 - The high speed download utility
 *
 * Copyright (C) 2006 Tatsuhiro Tsujikawa
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * In addition, as a special exception, the copyright holders give
 * permission to link the code of portions of this program with the
 * OpenSSL library under certain conditions as described in each
 * individual source file, and distribute linked combinations
 * including the two.
 * You must obey the GNU General Public License in all respects
 * for all of the code used other than OpenSSL.  If you modify
 * file(s) with this exception, you may extend this exception to your
 * version of the file(s), but you are not obligated to do so.  If you
 * do not wish to do so, delete this exception statement from your
 * version.  If you delete this exception statement from all source
 * files in the program, then also delete it here.
 */
/* copyright --> */
#include "util.h"

37 38 39
#ifdef __sun
// For opensolaris, just include signal.h which includes sys/signal.h
#ifdef HAVE_SIGNAL_H
40
#include <signal.h>
41
#endif // HAVE_SIGNAL_H
42
#else  // !__sun
43
#ifdef HAVE_SYS_SIGNAL_H
44
#include <sys/signal.h>
45 46
#endif // HAVE_SYS_SIGNAL_H
#ifdef HAVE_SIGNAL_H
47
#include <signal.h>
48 49 50
#endif // HAVE_SIGNAL_H
#endif // !__sun

51 52
#include <sys/types.h>
#ifdef HAVE_PWD_H
53
#include <pwd.h>
54
#endif // HAVE_PWD_H
55

56
#include <array>
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
#include <cerrno>
#include <cassert>
#include <cstring>
#include <cstdio>
#include <cstdlib>
#include <sstream>
#include <ostream>
#include <algorithm>
#include <fstream>
#include <iomanip>

#include "SimpleRandomizer.h"
#include "File.h"
#include "Randomizer.h"
#include "a2netcompat.h"
#include "BitfieldMan.h"
#include "DefaultDiskWriter.h"
#include "FatalException.h"
#include "FileEntry.h"
#include "A2STR.h"
#include "array_fun.h"
#include "bitfield.h"
#include "DownloadHandlerConstants.h"
#include "RequestGroup.h"
#include "LogFactory.h"
#include "Logger.h"
#include "Option.h"
#include "DownloadContext.h"
#include "BufferedFile.h"
#include "SocketCore.h"
87
#include "Lock.h"
88

89 90
#include "MessageDigest.h"
#include "message_digest_helper.h"
91 92 93

// For libc6 which doesn't define ULLONG_MAX properly because of broken limits.h
#ifndef ULLONG_MAX
94
#define ULLONG_MAX 18446744073709551615ULL
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
#endif // ULLONG_MAX

namespace aria2 {

#ifdef __MINGW32__
namespace {
int utf8ToWChar(wchar_t* out, size_t outLength, const char* src)
{
  return MultiByteToWideChar(CP_UTF8, 0, src, -1, out, outLength);
}
} // namespace

namespace {
int wCharToUtf8(char* out, size_t outLength, const wchar_t* src)
{
110 111
  return WideCharToMultiByte(CP_UTF8, 0, src, -1, out, outLength, nullptr,
                             nullptr);
112 113 114 115 116
}
} // namespace

std::wstring utf8ToWChar(const char* src)
{
117
  int len = utf8ToWChar(nullptr, 0, src);
118
  if (len <= 0) {
119 120
    abort();
  }
121 122
  auto buf = make_unique<wchar_t[]>((size_t)len);
  len = utf8ToWChar(buf.get(), len, src);
123
  if (len <= 0) {
124
    abort();
125 126
  }
  else {
127
    return buf.get();
128 129 130 131 132 133 134 135 136 137
  }
}

std::wstring utf8ToWChar(const std::string& src)
{
  return utf8ToWChar(src.c_str());
}

std::string wCharToUtf8(const std::wstring& wsrc)
{
138
  int len = wCharToUtf8(nullptr, 0, wsrc.c_str());
139
  if (len <= 0) {
140 141
    abort();
  }
142 143
  auto buf = make_unique<char[]>((size_t)len);
  len = wCharToUtf8(buf.get(), len, wsrc.c_str());
144
  if (len <= 0) {
145
    abort();
146 147
  }
  else {
148
    return buf.get();
149 150 151
  }
}

152 153 154 155 156 157 158 159
std::string toForwardSlash(const std::string& src)
{
  auto dst = src;
  std::transform(std::begin(dst), std::end(dst), std::begin(dst),
                 [](char c) { return c == '\\' ? '/' : c; });
  return dst;
}

160 161 162 163
#endif // __MINGW32__

namespace util {

164
const char DEFAULT_STRIP_CHARSET[] = "\r\n\t ";
165

166
std::string strip(const std::string& str, const char* chars)
167
{
168 169
  std::pair<std::string::const_iterator, std::string::const_iterator> p =
      stripIter(str.begin(), str.end(), chars);
170 171 172 173 174 175 176
  return std::string(p.first, p.second);
}

std::string itos(int64_t value, bool comma)
{
  bool flag = false;
  std::string str;
177 178 179
  if (value < 0) {
    if (value == INT64_MIN) {
      if (comma) {
180
        str = "-9,223,372,036,854,775,808";
181 182
      }
      else {
183 184 185 186 187 188 189 190
        str = "-9223372036854775808";
      }
      return str;
    }
    flag = true;
    value = -value;
  }
  str = uitos(value, comma);
191
  if (flag) {
192 193 194 195 196
    str.insert(str.begin(), '-');
  }
  return str;
}

197 198 199 200
int64_t difftv(struct timeval tv1, struct timeval tv2)
{
  if ((tv1.tv_sec < tv2.tv_sec) ||
      ((tv1.tv_sec == tv2.tv_sec) && (tv1.tv_usec < tv2.tv_usec))) {
201 202
    return 0;
  }
203 204
  return ((int64_t)(tv1.tv_sec - tv2.tv_sec) * 1000000 + tv1.tv_usec -
          tv2.tv_usec);
205 206
}

207 208 209
int32_t difftvsec(struct timeval tv1, struct timeval tv2)
{
  if (tv1.tv_sec < tv2.tv_sec) {
210 211
    return 0;
  }
212
  return tv1.tv_sec - tv2.tv_sec;
213 214
}

215 216 217 218
std::string replace(const std::string& target, const std::string& oldstr,
                    const std::string& newstr)
{
  if (target.empty() || oldstr.empty()) {
219 220 221 222 223
    return target;
  }
  std::string result;
  std::string::size_type p = 0;
  std::string::size_type np = target.find(oldstr);
224 225
  while (np != std::string::npos) {
    result.append(target.begin() + p, target.begin() + np);
226
    result += newstr;
227
    p = np + oldstr.size();
228 229
    np = target.find(oldstr, p);
  }
230
  result.append(target.begin() + p, target.end());
231 232 233 234 235 236 237 238
  return result;
}

bool isAlpha(const char c)
{
  return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z');
}

239
bool isDigit(const char c) { return '0' <= c && c <= '9'; }
240 241 242 243 244 245 246 247

bool isHexDigit(const char c)
{
  return isDigit(c) || ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f');
}

bool isHexDigit(const std::string& s)
{
248
  for (const auto& c : s) {
249
    if (!isHexDigit(c)) {
250 251 252 253 254 255 256 257
      return false;
    }
  }
  return true;
}

bool inRFC3986ReservedChars(const char c)
{
258 259 260 261
  static const char reserved[] = {':', '/',  '?', '#', '[', ']', '@', '!', '$',
                                  '&', '\'', '(', ')', '*', '+', ',', ';', '='};
  return std::find(std::begin(reserved), std::end(reserved), c) !=
         std::end(reserved);
262 263 264 265
}

bool inRFC3986UnreservedChars(const char c)
{
266
  static const char unreserved[] = {'-', '.', '_', '~'};
267
  return isAlpha(c) || isDigit(c) ||
268 269
         std::find(std::begin(unreserved), std::end(unreserved), c) !=
             std::end(unreserved);
270 271 272 273
}

bool inRFC2978MIMECharset(const char c)
{
274 275
  static const char chars[] = {'!', '#', '$', '%', '&', '\'', '+',
                               '-', '^', '_', '`', '{', '}',  '~'};
276
  return isAlpha(c) || isDigit(c) ||
277
         std::find(std::begin(chars), std::end(chars), c) != std::end(chars);
278 279 280 281
}

bool inRFC2616HttpToken(const char c)
{
282 283
  static const char chars[] = {'!', '#', '$', '%', '&', '\'', '*', '+',
                               '-', '.', '^', '_', '`', '|',  '~'};
284
  return isAlpha(c) || isDigit(c) ||
285
         std::find(std::begin(chars), std::end(chars), c) != std::end(chars);
286 287
}

288 289 290 291 292 293 294 295 296 297 298
bool inRFC5987AttrChar(const char c)
{
  return inRFC2616HttpToken(c) && c != '*' && c != '\'' && c != '%';
}

// Returns nonzero if |c| is in ISO/IEC 8859-1 character set.
bool isIso8859p1(unsigned char c)
{
  return (0x20u <= c && c <= 0x7eu) || 0xa0u <= c;
}

299 300
bool isLws(const char c) { return c == ' ' || c == '\t'; }
bool isCRLF(const char c) { return c == '\r' || c == '\n'; }
301

302
namespace {
303

304
bool isUtf8Tail(unsigned char ch) { return in(ch, 0x80u, 0xbfu); }
305 306 307 308

bool inPercentEncodeMini(const unsigned char c)
{
  return c > 0x20 && c < 0x7fu &&
309 310
         // Chromium escapes following characters. Firefox4 escapes more.
         c != '"' && c != '<' && c != '>';
311 312
}

313 314 315 316
} // namespace

bool isUtf8(const std::string& str)
{
317 318
  for (std::string::const_iterator s = str.begin(), eos = str.end(); s != eos;
       ++s) {
319 320
    unsigned char firstChar = *s;
    // See ABNF in http://tools.ietf.org/search/rfc3629#section-4
321 322 323 324 325
    if (in(firstChar, 0x20u, 0x7eu) || firstChar == 0x08u || // \b
        firstChar == 0x09u ||                                // \t
        firstChar == 0x0au ||                                // \n
        firstChar == 0x0cu ||                                // \f
        firstChar == 0x0du                                   // \r
326
    ) {
327
      // UTF8-1 (without ctrl chars)
328 329
    }
    else if (in(firstChar, 0xc2u, 0xdfu)) {
330
      // UTF8-2
331
      if (++s == eos || !isUtf8Tail(*s)) {
332 333
        return false;
      }
334 335
    }
    else if (0xe0u == firstChar) {
336
      // UTF8-3
337 338
      if (++s == eos || !in(static_cast<unsigned char>(*s), 0xa0u, 0xbfu) ||
          ++s == eos || !isUtf8Tail(*s)) {
339 340
        return false;
      }
341 342
    }
    else if (in(firstChar, 0xe1u, 0xecu) || in(firstChar, 0xeeu, 0xefu)) {
343
      // UTF8-3
344
      if (++s == eos || !isUtf8Tail(*s) || ++s == eos || !isUtf8Tail(*s)) {
345 346
        return false;
      }
347 348
    }
    else if (0xedu == firstChar) {
349
      // UTF8-3
350 351
      if (++s == eos || !in(static_cast<unsigned char>(*s), 0x80u, 0x9fu) ||
          ++s == eos || !isUtf8Tail(*s)) {
352
        return false;
353
      }
354 355
    }
    else if (0xf0u == firstChar) {
356
      // UTF8-4
357 358
      if (++s == eos || !in(static_cast<unsigned char>(*s), 0x90u, 0xbfu) ||
          ++s == eos || !isUtf8Tail(*s) || ++s == eos || !isUtf8Tail(*s)) {
359 360
        return false;
      }
361 362
    }
    else if (in(firstChar, 0xf1u, 0xf3u)) {
363
      // UTF8-4
364 365
      if (++s == eos || !isUtf8Tail(*s) || ++s == eos || !isUtf8Tail(*s) ||
          ++s == eos || !isUtf8Tail(*s)) {
366 367
        return false;
      }
368 369
    }
    else if (0xf4u == firstChar) {
370
      // UTF8-4
371 372
      if (++s == eos || !in(static_cast<unsigned char>(*s), 0x80u, 0x8fu) ||
          ++s == eos || !isUtf8Tail(*s) || ++s == eos || !isUtf8Tail(*s)) {
373 374
        return false;
      }
375 376
    }
    else {
377 378 379 380 381 382 383 384 385
      return false;
    }
  }
  return true;
}

std::string percentEncode(const unsigned char* target, size_t len)
{
  std::string dest;
386 387
  for (size_t i = 0; i < len; ++i) {
    if (inRFC3986UnreservedChars(target[i])) {
388
      dest += target[i];
389 390
    }
    else {
391 392 393 394 395 396 397 398
      dest.append(fmt("%%%02X", target[i]));
    }
  }
  return dest;
}

std::string percentEncode(const std::string& target)
{
399 400 401 402
  if (std::find_if_not(target.begin(), target.end(),
                       inRFC3986UnreservedChars) == target.end()) {
    return target;
  }
403 404 405 406 407 408
  return percentEncode(reinterpret_cast<const unsigned char*>(target.c_str()),
                       target.size());
}

std::string percentEncodeMini(const std::string& src)
{
409 410 411 412
  if (std::find_if_not(src.begin(), src.end(), inPercentEncodeMini) ==
      src.end()) {
    return src;
  }
413
  std::string result;
414 415
  for (auto c : src) {
    if (!inPercentEncodeMini(c)) {
416
      result += fmt("%%%02X", static_cast<unsigned char>(c));
417 418
    }
    else {
419 420 421 422 423 424
      result += c;
    }
  }
  return result;
}

425 426
std::string torrentPercentEncode(const unsigned char* target, size_t len)
{
427
  std::string dest;
428 429
  for (size_t i = 0; i < len; ++i) {
    if (isAlpha(target[i]) || isDigit(target[i])) {
430
      dest += target[i];
431 432
    }
    else {
433 434 435 436 437 438 439 440
      dest.append(fmt("%%%02X", target[i]));
    }
  }
  return dest;
}

std::string torrentPercentEncode(const std::string& target)
{
441 442
  return torrentPercentEncode(
      reinterpret_cast<const unsigned char*>(target.c_str()), target.size());
443 444
}

445 446
std::string percentDecode(std::string::const_iterator first,
                          std::string::const_iterator last)
447 448
{
  std::string result;
449 450 451 452 453 454
  for (; first != last; ++first) {
    if (*first == '%') {
      if (first + 1 != last && first + 2 != last && isHexDigit(*(first + 1)) &&
          isHexDigit(*(first + 2))) {
        result +=
            hexCharToUInt(*(first + 1)) * 16 + hexCharToUInt(*(first + 2));
455
        first += 2;
456 457
      }
      else {
458 459
        result += *first;
      }
460 461
    }
    else {
462 463 464 465 466 467
      result += *first;
    }
  }
  return result;
}

468 469 470
std::string toHex(const unsigned char* src, size_t len)
{
  std::string out(len * 2, '\0');
471
  std::string::iterator o = out.begin();
472 473
  const unsigned char* last = src + len;
  for (const unsigned char* i = src; i != last; ++i) {
474
    *o = (*i >> 4);
475 476 477
    *(o + 1) = (*i) & 0x0fu;
    for (int j = 0; j < 2; ++j) {
      if (*o < 10) {
478
        *o += '0';
479 480 481
      }
      else {
        *o += 'a' - 10;
482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
      }
      ++o;
    }
  }
  return out;
}

std::string toHex(const char* src, size_t len)
{
  return toHex(reinterpret_cast<const unsigned char*>(src), len);
}

std::string toHex(const std::string& src)
{
  return toHex(reinterpret_cast<const unsigned char*>(src.c_str()), src.size());
}

unsigned int hexCharToUInt(unsigned char ch)
{
501
  if ('a' <= ch && ch <= 'f') {
502 503
    ch -= 'a';
    ch += 10;
504 505
  }
  else if ('A' <= ch && ch <= 'F') {
506 507
    ch -= 'A';
    ch += 10;
508 509
  }
  else if ('0' <= ch && ch <= '9') {
510
    ch -= '0';
511 512
  }
  else {
513 514 515 516 517
    ch = 255;
  }
  return ch;
}

518 519
std::string secfmt(time_t sec)
{
520 521
  time_t tsec = sec;
  std::string str;
522 523
  if (sec >= 3600) {
    str = fmt("%" PRId64 "h", static_cast<int64_t>(sec / 3600));
524 525
    sec %= 3600;
  }
526 527
  if (sec >= 60) {
    str += fmt("%dm", static_cast<int>(sec / 60));
528 529
    sec %= 60;
  }
530
  if (sec || tsec == 0) {
531 532 533 534 535 536
    str += fmt("%ds", static_cast<int>(sec));
  }
  return str;
}

namespace {
537
template <typename T, typename F>
538 539
bool parseLong(T& res, F f, const std::string& s, int base)
{
540
  if (s.empty()) {
541 542 543 544 545
    return false;
  }
  char* endptr;
  errno = 0;
  res = f(s.c_str(), &endptr, base);
546
  if (errno == ERANGE) {
547 548
    return false;
  }
549
  if (*endptr != '\0') {
550
    for (const char *i = endptr, *eoi = s.c_str() + s.size(); i < eoi; ++i) {
551
      if (!isspace(*i)) {
552 553 554 555 556 557 558 559 560 561 562
        return false;
      }
    }
  }
  return true;
}
} // namespace

bool parseIntNoThrow(int32_t& res, const std::string& s, int base)
{
  long int t;
563 564 565
  if (parseLong(t, strtol, s, base) &&
      t >= std::numeric_limits<int32_t>::min() &&
      t <= std::numeric_limits<int32_t>::max()) {
566 567
    res = t;
    return true;
568 569
  }
  else {
570 571 572 573 574 575 576
    return false;
  }
}

bool parseUIntNoThrow(uint32_t& res, const std::string& s, int base)
{
  long int t;
577 578
  if (parseLong(t, strtol, s, base) && t >= 0 &&
      t <= std::numeric_limits<int32_t>::max()) {
579 580
    res = t;
    return true;
581 582
  }
  else {
583 584 585 586 587 588
    return false;
  }
}

bool parseLLIntNoThrow(int64_t& res, const std::string& s, int base)
{
589 590
  int64_t t;
  if (parseLong(t, strtoll, s, base)) {
591 592
    res = t;
    return true;
593 594
  }
  else {
595 596 597 598
    return false;
  }
}

599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
bool parseDoubleNoThrow(double& res, const std::string& s)
{
  if (s.empty()) {
    return false;
  }

  errno = 0;
  char* endptr;
  auto d = strtod(s.c_str(), &endptr);

  if (errno == ERANGE) {
    return false;
  }

  if (endptr != s.c_str() + s.size()) {
    for (auto i = std::begin(s) + (endptr - s.c_str()); i != std::end(s); ++i) {
      if (!isspace(*i)) {
        return false;
      }
    }
  }

  res = d;

  return true;
}

626
SegList<int> parseIntSegments(const std::string& src)
627
{
628
  SegList<int> sgl;
629 630
  for (std::string::const_iterator i = src.begin(), eoi = src.end();
       i != eoi;) {
631
    std::string::const_iterator j = std::find(i, eoi, ',');
632
    if (j == i) {
633 634 635 636
      ++i;
      continue;
    }
    std::string::const_iterator p = std::find(i, j, '-');
637
    if (p == j) {
638
      int a;
639 640 641 642
      if (parseIntNoThrow(a, std::string(i, j))) {
        sgl.add(a, a + 1);
      }
      else {
643 644
        throw DL_ABORT_EX(fmt("Bad range %s", std::string(i, j).c_str()));
      }
645 646
    }
    else if (p == i || p + 1 == j) {
647
      throw DL_ABORT_EX(fmt(MSG_INCOMPLETE_RANGE, std::string(i, j).c_str()));
648 649
    }
    else {
650
      int a, b;
651 652 653 654 655
      if (parseIntNoThrow(a, std::string(i, p)) &&
          parseIntNoThrow(b, (std::string(p + 1, j)))) {
        sgl.add(a, b + 1);
      }
      else {
656 657
        throw DL_ABORT_EX(fmt("Bad range %s", std::string(i, j).c_str()));
      }
658
    }
659
    if (j == eoi) {
660 661
      break;
    }
662
    i = j + 1;
663
  }
664
  return sgl;
665 666 667
}

namespace {
668 669 670 671
void computeHeadPieces(
    std::vector<size_t>& indexes,
    const std::vector<std::shared_ptr<FileEntry>>& fileEntries,
    size_t pieceLength, int64_t head)
672
{
673
  if (head == 0) {
674 675
    return;
  }
676 677
  for (const auto& fi : fileEntries) {
    if (fi->getLength() == 0) {
678 679
      continue;
    }
680 681 682
    const size_t lastIndex =
        (fi->getOffset() + std::min(head, fi->getLength()) - 1) / pieceLength;
    for (size_t idx = fi->getOffset() / pieceLength; idx <= lastIndex; ++idx) {
683
      indexes.push_back(idx);
684 685 686 687 688 689
    }
  }
}
} // namespace

namespace {
690 691 692 693
void computeTailPieces(
    std::vector<size_t>& indexes,
    const std::vector<std::shared_ptr<FileEntry>>& fileEntries,
    size_t pieceLength, int64_t tail)
694
{
695
  if (tail == 0) {
696 697
    return;
  }
698 699
  for (const auto& fi : fileEntries) {
    if (fi->getLength() == 0) {
700 701
      continue;
    }
702
    int64_t endOffset = fi->getLastOffset();
703 704
    size_t fromIndex =
        (endOffset - 1 - (std::min(tail, fi->getLength()) - 1)) / pieceLength;
705 706 707
    const size_t toIndex = (endOffset - 1) / pieceLength;
    while (fromIndex <= toIndex) {
      indexes.push_back(fromIndex++);
708 709 710 711 712
    }
  }
}
} // namespace

713 714 715 716
void parsePrioritizePieceRange(
    std::vector<size_t>& result, const std::string& src,
    const std::vector<std::shared_ptr<FileEntry>>& fileEntries,
    size_t pieceLength, int64_t defaultSize)
717 718 719 720
{
  std::vector<size_t> indexes;
  std::vector<Scip> parts;
  splitIter(src.begin(), src.end(), std::back_inserter(parts), ',', true);
721 722
  for (const auto& i : parts) {
    if (util::streq(i.first, i.second, "head")) {
723
      computeHeadPieces(indexes, fileEntries, pieceLength, defaultSize);
724
    }
725
    else if (util::startsWith(i.first, i.second, "head=")) {
726
      std::string sizestr(i.first + 5, i.second);
727 728
      computeHeadPieces(indexes, fileEntries, pieceLength,
                        std::max((int64_t)0, getRealSize(sizestr)));
729
    }
730
    else if (util::streq(i.first, i.second, "tail")) {
731
      computeTailPieces(indexes, fileEntries, pieceLength, defaultSize);
732
    }
733
    else if (util::startsWith(i.first, i.second, "tail=")) {
734
      std::string sizestr(i.first + 5, i.second);
735 736
      computeTailPieces(indexes, fileEntries, pieceLength,
                        std::max((int64_t)0, getRealSize(sizestr)));
737 738
    }
    else {
739 740
      throw DL_ABORT_EX(
          fmt("Unrecognized token %s", std::string(i.first, i.second).c_str()));
741 742 743 744 745 746 747 748 749
    }
  }
  std::sort(indexes.begin(), indexes.end());
  indexes.erase(std::unique(indexes.begin(), indexes.end()), indexes.end());
  result.insert(result.end(), indexes.begin(), indexes.end());
}

// Converts ISO/IEC 8859-1 string to UTF-8 string.  If there is a
// character not in ISO/IEC 8859-1, returns empty string.
750
std::string iso8859p1ToUtf8(const char* src, size_t len)
751 752
{
  std::string dest;
753
  for (const char *p = src, *last = src + len; p != last; ++p) {
754
    unsigned char c = *p;
755 756
    if (0xa0u <= c) {
      if (c <= 0xbfu) {
757
        dest += 0xc2u;
758 759
      }
      else {
760 761
        dest += 0xc3u;
      }
762 763 764
      dest += c & (~0x40u);
    }
    else if (0x80u <= c && c <= 0x9fu) {
765
      return "";
766 767
    }
    else {
768 769 770 771 772 773
      dest += c;
    }
  }
  return dest;
}

774 775 776
std::string iso8859p1ToUtf8(const std::string& src)
{
  return iso8859p1ToUtf8(src.c_str(), src.size());
777 778
}

779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
/* Start of utf8 dfa */
/* Copyright (c) 2008-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
 * See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
 *
 * Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
 *
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
 * files (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy,
 * modify, merge, publish, distribute, sublicense, and/or sell copies
 * of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#define UTF8_ACCEPT 0
#define UTF8_REJECT 12

static const uint8_t utf8d[] = {
809 810 811 812
    /*
     * The first part of the table maps bytes to character classes that
     * to reduce the size of the transition table and create bitmasks.
     */
813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    9,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    7,
    8,
    8,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    2,
    10,
    3,
    3,
    3,
    3,
    3,
    3,
    3,
    3,
    3,
    3,
    3,
    3,
    4,
    3,
    3,
    11,
    6,
    6,
    6,
    5,
    8,
    8,
    8,
    8,
    8,
    8,
    8,
    8,
    8,
    8,
    8,
1069 1070 1071 1072 1073

    /*
     * The second part is a transition table that maps a combination
     * of a state of the automaton and a character class to a state.
     */
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
    0,
    12,
    24,
    36,
    60,
    96,
    84,
    12,
    12,
    12,
    48,
    72,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    0,
    12,
    12,
    12,
    12,
    12,
    0,
    12,
    0,
    12,
    12,
    12,
    24,
    12,
    12,
    12,
    12,
    12,
    24,
    12,
    24,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    24,
    12,
    12,
    12,
    12,
    12,
    24,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    24,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    36,
    12,
    36,
    12,
    12,
    12,
    36,
    12,
    12,
    12,
    12,
    12,
    36,
    12,
    36,
    12,
    12,
    12,
    36,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
    12,
1182 1183
};

1184 1185
static uint32_t utf8dfa(uint32_t* state, uint32_t* codep, uint32_t byte)
{
1186 1187
  uint32_t type = utf8d[byte];

1188 1189
  *codep = (*state != UTF8_ACCEPT) ? (byte & 0x3fu) | (*codep << 6)
                                   : (0xff >> type) & (byte);
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226

  *state = utf8d[256 + *state + type];
  return *state;
}

/* End of utf8 dfa */

typedef enum {
  CD_BEFORE_DISPOSITION_TYPE,
  CD_AFTER_DISPOSITION_TYPE,
  CD_DISPOSITION_TYPE,
  CD_BEFORE_DISPOSITION_PARM_NAME,
  CD_AFTER_DISPOSITION_PARM_NAME,
  CD_DISPOSITION_PARM_NAME,
  CD_BEFORE_VALUE,
  CD_AFTER_VALUE,
  CD_QUOTED_STRING,
  CD_TOKEN,
  CD_BEFORE_EXT_VALUE,
  CD_CHARSET,
  CD_LANGUAGE,
  CD_VALUE_CHARS,
  CD_VALUE_CHARS_PCT_ENCODED1,
  CD_VALUE_CHARS_PCT_ENCODED2
} content_disposition_parse_state;

typedef enum {
  CD_FILENAME_FOUND = 1,
  CD_EXT_FILENAME_FOUND = 1 << 1
} content_disposition_parse_flag;

typedef enum {
  CD_ENC_UNKNOWN,
  CD_ENC_UTF8,
  CD_ENC_ISO_8859_1
} content_disposition_charset;

1227 1228
ssize_t parse_content_disposition(char* dest, size_t destlen,
                                  const char** charsetp, size_t* charsetlenp,
1229
                                  const char* in, size_t len, bool defaultUTF8)
1230
{
1231 1232
  const char *p = in, *eop = in + len, *mark_first = nullptr,
             *mark_last = nullptr;
1233 1234 1235 1236 1237 1238
  int state = CD_BEFORE_DISPOSITION_TYPE;
  int in_file_parm = 0;
  int flags = 0;
  int quoted_seen = 0;
  int charset = 0;
  /* To suppress warnings */
1239
  char* dp = dest;
1240
  size_t dlen = destlen;
1241
  uint32_t dfa_state = UTF8_ACCEPT;
1242 1243 1244
  uint32_t dfa_code = 0;
  uint8_t pctval = 0;

1245
  *charsetp = nullptr;
1246 1247
  *charsetlenp = 0;

1248 1249
  for (; p != eop; ++p) {
    switch (state) {
1250
    case CD_BEFORE_DISPOSITION_TYPE:
1251
      if (inRFC2616HttpToken(*p)) {
1252
        state = CD_DISPOSITION_TYPE;
1253 1254
      }
      else if (!isLws(*p)) {
1255
        return -1;
1256
      }
1257 1258 1259
      break;
    case CD_AFTER_DISPOSITION_TYPE:
    case CD_DISPOSITION_TYPE:
1260
      if (*p == ';') {
1261
        state = CD_BEFORE_DISPOSITION_PARM_NAME;
1262 1263
      }
      else if (isLws(*p)) {
1264
        state = CD_AFTER_DISPOSITION_TYPE;
1265 1266
      }
      else if (state == CD_AFTER_DISPOSITION_TYPE || !inRFC2616HttpToken(*p)) {
1267
        return -1;
1268
      }
1269 1270
      break;
    case CD_BEFORE_DISPOSITION_PARM_NAME:
1271
      if (inRFC2616HttpToken(*p)) {
1272 1273
        mark_first = p;
        state = CD_DISPOSITION_PARM_NAME;
1274 1275
      }
      else if (!isLws(*p)) {
1276
        return -1;
1277
      }
1278 1279 1280
      break;
    case CD_AFTER_DISPOSITION_PARM_NAME:
    case CD_DISPOSITION_PARM_NAME:
1281 1282
      if (*p == '=') {
        if (state == CD_DISPOSITION_PARM_NAME) {
1283 1284 1285
          mark_last = p;
        }
        in_file_parm = 0;
1286 1287
        if (strieq(mark_first, mark_last, "filename*")) {
          if ((flags & CD_EXT_FILENAME_FOUND) == 0) {
1288
            in_file_parm = 1;
1289 1290
          }
          else {
1291 1292 1293
            return -1;
          }
          state = CD_BEFORE_EXT_VALUE;
1294 1295 1296
        }
        else if (strieq(mark_first, mark_last, "filename")) {
          if (flags & CD_FILENAME_FOUND) {
1297
            return -1;
1298
          }
1299
          if ((flags & CD_EXT_FILENAME_FOUND) == 0) {
1300 1301 1302
            in_file_parm = 1;
          }
          state = CD_BEFORE_VALUE;
1303 1304
        }
        else {
1305
          /* ext-token must be characters in token, followed by "*" */
1306
          if (mark_first != mark_last - 1 && *(mark_last - 1) == '*') {
1307
            state = CD_BEFORE_EXT_VALUE;
1308 1309
          }
          else {
1310
            state = CD_BEFORE_VALUE;
1311 1312
          }
        }
1313
        if (in_file_parm) {
1314 1315 1316
          dp = dest;
          dlen = destlen;
        }
1317 1318
      }
      else if (isLws(*p)) {
1319 1320
        mark_last = p;
        state = CD_AFTER_DISPOSITION_PARM_NAME;
1321 1322 1323
      }
      else if (state == CD_AFTER_DISPOSITION_PARM_NAME ||
               !inRFC2616HttpToken(*p)) {
1324
        return -1;
1325
      }
1326 1327
      break;
    case CD_BEFORE_VALUE:
1328
      if (*p == '"') {
1329 1330
        quoted_seen = 0;
        state = CD_QUOTED_STRING;
1331 1332 1333 1334
        if (defaultUTF8) {
          dfa_state = UTF8_ACCEPT;
          dfa_code = 0;
        }
1335 1336 1337 1338
      }
      else if (inRFC2616HttpToken(*p)) {
        if (in_file_parm) {
          if (dlen == 0) {
1339
            return -1;
1340 1341
          }
          else {
1342 1343 1344 1345 1346
            *dp++ = *p;
            --dlen;
          }
        }
        state = CD_TOKEN;
1347 1348
      }
      else if (!isLws(*p)) {
1349 1350 1351 1352
        return -1;
      }
      break;
    case CD_AFTER_VALUE:
1353
      if (*p == ';') {
1354
        state = CD_BEFORE_DISPOSITION_PARM_NAME;
1355 1356
      }
      else if (!isLws(*p)) {
1357 1358 1359 1360
        return -1;
      }
      break;
    case CD_QUOTED_STRING:
1361
      if (*p == '\\' && quoted_seen == 0) {
1362
        quoted_seen = 1;
1363 1364
      }
      else if (*p == '"' && quoted_seen == 0) {
1365 1366 1367
        if (defaultUTF8 && dfa_state != UTF8_ACCEPT) {
          return -1;
        }
1368
        if (in_file_parm) {
1369 1370 1371
          flags |= CD_FILENAME_FOUND;
        }
        state = CD_AFTER_VALUE;
1372 1373
      }
      else {
1374 1375
        /* TEXT which is OCTET except CTLs, but including LWS. Accept
           ISO-8859-1 chars, or UTF-8 if defaultUTF8 is set */
1376
        quoted_seen = 0;
1377 1378 1379 1380 1381 1382 1383
        if (defaultUTF8) {
          if (utf8dfa(&dfa_state, &dfa_code, (unsigned char)*p) ==
              UTF8_REJECT) {
            return -1;
          }
        }
        else if (!isIso8859p1(*p)) {
1384 1385
          return -1;
        }
1386 1387
        if (in_file_parm) {
          if (dlen == 0) {
1388
            return -1;
1389 1390
          }
          else {
1391 1392 1393 1394
            *dp++ = *p;
            --dlen;
          }
        }
1395
      }
1396 1397
      break;
    case CD_TOKEN:
1398 1399 1400
      if (inRFC2616HttpToken(*p)) {
        if (in_file_parm) {
          if (dlen == 0) {
1401
            return -1;
1402 1403
          }
          else {
1404 1405 1406 1407
            *dp++ = *p;
            --dlen;
          }
        }
1408 1409 1410
      }
      else if (*p == ';') {
        if (in_file_parm) {
1411 1412 1413
          flags |= CD_FILENAME_FOUND;
        }
        state = CD_BEFORE_DISPOSITION_PARM_NAME;
1414 1415 1416
      }
      else if (isLws(*p)) {
        if (in_file_parm) {
1417 1418 1419
          flags |= CD_FILENAME_FOUND;
        }
        state = CD_AFTER_VALUE;
1420 1421
      }
      else {
1422
        return -1;
1423
      }
1424 1425
      break;
    case CD_BEFORE_EXT_VALUE:
1426
      if (*p == '\'') {
1427 1428
        /* Empty charset is not allowed */
        return -1;
1429 1430
      }
      else if (inRFC2978MIMECharset(*p)) {
1431 1432
        mark_first = p;
        state = CD_CHARSET;
1433 1434
      }
      else if (!isLws(*p)) {
1435
        return -1;
1436
      }
1437 1438
      break;
    case CD_CHARSET:
1439
      if (*p == '\'') {
1440 1441 1442
        mark_last = p;
        *charsetp = mark_first;
        *charsetlenp = mark_last - mark_first;
1443
        if (strieq(mark_first, mark_last, "utf-8")) {
1444 1445 1446
          charset = CD_ENC_UTF8;
          dfa_state = UTF8_ACCEPT;
          dfa_code = 0;
1447 1448
        }
        else if (strieq(mark_first, mark_last, "iso-8859-1")) {
1449
          charset = CD_ENC_ISO_8859_1;
1450 1451
        }
        else {
1452 1453 1454
          charset = CD_ENC_UNKNOWN;
        }
        state = CD_LANGUAGE;
1455 1456
      }
      else if (!inRFC2978MIMECharset(*p)) {
1457
        return -1;
1458
      }
1459 1460
      break;
    case CD_LANGUAGE:
1461 1462
      if (*p == '\'') {
        if (in_file_parm) {
1463 1464 1465 1466
          dp = dest;
          dlen = destlen;
        }
        state = CD_VALUE_CHARS;
1467 1468
      }
      else if (*p != '-' && !isAlpha(*p) && !isDigit(*p)) {
1469
        return -1;
1470
      }
1471 1472
      break;
    case CD_VALUE_CHARS:
1473 1474
      if (inRFC5987AttrChar(*p)) {
        if (charset == CD_ENC_UTF8) {
1475 1476
          if (utf8dfa(&dfa_state, &dfa_code, static_cast<unsigned char>(*p)) ==
              UTF8_REJECT) {
1477 1478 1479
            return -1;
          }
        }
1480 1481
        if (in_file_parm) {
          if (dlen == 0) {
1482
            return -1;
1483 1484
          }
          else {
1485 1486 1487 1488
            *dp++ = *p;
            --dlen;
          }
        }
1489 1490 1491 1492
      }
      else if (*p == '%') {
        if (in_file_parm) {
          if (dlen == 0) {
1493 1494 1495 1496 1497
            return -1;
          }
        }
        pctval = 0;
        state = CD_VALUE_CHARS_PCT_ENCODED1;
1498 1499 1500
      }
      else if (*p == ';' || isLws(*p)) {
        if (charset == CD_ENC_UTF8 && dfa_state != UTF8_ACCEPT) {
1501 1502
          return -1;
        }
1503
        if (in_file_parm) {
1504 1505
          flags |= CD_EXT_FILENAME_FOUND;
        }
1506
        if (*p == ';') {
1507
          state = CD_BEFORE_DISPOSITION_PARM_NAME;
1508 1509
        }
        else {
1510 1511
          state = CD_AFTER_VALUE;
        }
1512 1513
      }
      else if (!inRFC5987AttrChar(*p)) {
1514
        return -1;
1515
      }
1516 1517
      break;
    case CD_VALUE_CHARS_PCT_ENCODED1:
1518
      if (isHexDigit(*p)) {
1519 1520
        pctval |= hexCharToUInt(*p) << 4;
        state = CD_VALUE_CHARS_PCT_ENCODED2;
1521 1522
      }
      else {
1523
        return -1;
1524
      }
1525 1526
      break;
    case CD_VALUE_CHARS_PCT_ENCODED2:
1527
      if (isHexDigit(*p)) {
1528
        pctval |= hexCharToUInt(*p);
1529 1530
        if (charset == CD_ENC_UTF8) {
          if (utf8dfa(&dfa_state, &dfa_code, pctval) == UTF8_REJECT) {
1531 1532
            return -1;
          }
1533 1534 1535
        }
        else if (charset == CD_ENC_ISO_8859_1) {
          if (!isIso8859p1(pctval)) {
1536 1537 1538
            return -1;
          }
        }
1539
        if (in_file_parm) {
1540 1541 1542 1543
          *dp++ = pctval;
          --dlen;
        }
        state = CD_VALUE_CHARS;
1544 1545
      }
      else {
1546
        return -1;
1547
      }
1548 1549 1550
      break;
    }
  }
1551
  switch (state) {
1552 1553 1554 1555 1556
  case CD_BEFORE_DISPOSITION_TYPE:
  case CD_AFTER_DISPOSITION_TYPE:
  case CD_DISPOSITION_TYPE:
  case CD_AFTER_VALUE:
  case CD_TOKEN:
1557
    return destlen - dlen;
1558
  case CD_VALUE_CHARS:
1559
    if (charset == CD_ENC_UTF8 && dfa_state != UTF8_ACCEPT) {
1560
      return -1;
1561
    }
1562 1563 1564
    return destlen - dlen;
  default:
    return -1;
1565 1566 1567
  }
}

1568 1569
std::string getContentDispositionFilename(const std::string& header,
                                          bool defaultUTF8)
1570
{
1571 1572
  std::array<char, 1_k> cdval;
  size_t cdvallen = cdval.size();
1573 1574
  const char* charset;
  size_t charsetlen;
1575 1576
  ssize_t rv =
      parse_content_disposition(cdval.data(), cdvallen, &charset, &charsetlen,
1577
                                header.c_str(), header.size(), defaultUTF8);
1578
  if (rv == -1) {