libgo: update to Go1.10beta1
Update the Go library to the 1.10beta1 release. Requires a few changes to the compiler for modifications to the map runtime code, and to handle some nowritebarrier cases in the runtime. Reviewed-on: https://go-review.googlesource.com/86455 gotools/: * Makefile.am (go_cmd_vet_files): New variable. (go_cmd_buildid_files, go_cmd_test2json_files): New variables. (s-zdefaultcc): Change from constants to functions. (noinst_PROGRAMS): Add vet, buildid, and test2json. (cgo$(EXEEXT)): Link against $(LIBGOTOOL). (vet$(EXEEXT)): New target. (buildid$(EXEEXT)): New target. (test2json$(EXEEXT)): New target. (install-exec-local): Install all $(noinst_PROGRAMS). (uninstall-local): Uninstasll all $(noinst_PROGRAMS). (check-go-tool): Depend on $(noinst_PROGRAMS). Copy down objabi.go. (check-runtime): Depend on $(noinst_PROGRAMS). (check-cgo-test, check-carchive-test): Likewise. (check-vet): New target. (check): Depend on check-vet. Look at cmd_vet-testlog. (.PHONY): Add check-vet. * Makefile.in: Rebuild. From-SVN: r256365
This commit is contained in:
parent
8799df67f2
commit
1a2f01efa6
|
@ -1,4 +1,4 @@
|
|||
1319f36ccc65cf802b8e17ddd3d2da3ca6d82f4c
|
||||
dbc0c7e4329aada2ae3554c20cfb8cfa48041213
|
||||
|
||||
The first line of this file holds the git revision number of the last
|
||||
merge done from the gofrontend repository.
|
||||
|
|
|
@ -7483,6 +7483,7 @@ Builtin_call_expression::lower_make(Statement_inserter* inserter)
|
|||
return Expression::make_error(this->location());
|
||||
}
|
||||
len_arg = Expression::make_integer_ul(0, NULL, loc);
|
||||
len_small = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -7551,9 +7552,23 @@ Builtin_call_expression::lower_make(Statement_inserter* inserter)
|
|||
else if (is_map)
|
||||
{
|
||||
Expression* type_arg = Expression::make_type_descriptor(type, type_loc);
|
||||
call = Runtime::make_call(Runtime::MAKEMAP, loc, 4, type_arg, len_arg,
|
||||
Expression::make_nil(loc),
|
||||
Expression::make_nil(loc));
|
||||
if (!len_small)
|
||||
call = Runtime::make_call(Runtime::MAKEMAP64, loc, 3, type_arg,
|
||||
len_arg,
|
||||
Expression::make_nil(loc));
|
||||
else
|
||||
{
|
||||
Numeric_constant nclen;
|
||||
unsigned long vlen;
|
||||
if (len_arg->numeric_constant_value(&nclen)
|
||||
&& nclen.to_unsigned_long(&vlen) == Numeric_constant::NC_UL_VALID
|
||||
&& vlen <= Map_type::bucket_size)
|
||||
call = Runtime::make_call(Runtime::MAKEMAP_SMALL, loc, 0);
|
||||
else
|
||||
call = Runtime::make_call(Runtime::MAKEMAP, loc, 3, type_arg,
|
||||
len_arg,
|
||||
Expression::make_nil(loc));
|
||||
}
|
||||
}
|
||||
else if (is_chan)
|
||||
{
|
||||
|
@ -9503,14 +9518,8 @@ Call_expression::do_lower(Gogo* gogo, Named_object* function,
|
|||
// could implement them in normal code, but then we would have to
|
||||
// explicitly unwind the stack. These functions are intended to be
|
||||
// efficient. Note that this technique obviously only works for
|
||||
// direct calls, but that is the only way they are used. The actual
|
||||
// argument to these functions is always the address of a parameter;
|
||||
// we don't need that for the GCC builtin functions, so we just
|
||||
// ignore it.
|
||||
if (gogo->compiling_runtime()
|
||||
&& this->args_ != NULL
|
||||
&& this->args_->size() == 1
|
||||
&& gogo->package_name() == "runtime")
|
||||
// direct calls, but that is the only way they are used.
|
||||
if (gogo->compiling_runtime() && gogo->package_name() == "runtime")
|
||||
{
|
||||
Func_expression* fe = this->fn_->func_expression();
|
||||
if (fe != NULL
|
||||
|
@ -9518,15 +9527,21 @@ Call_expression::do_lower(Gogo* gogo, Named_object* function,
|
|||
&& fe->named_object()->package() == NULL)
|
||||
{
|
||||
std::string n = Gogo::unpack_hidden_name(fe->named_object()->name());
|
||||
if (n == "getcallerpc")
|
||||
if ((this->args_ == NULL || this->args_->size() == 0)
|
||||
&& n == "getcallerpc")
|
||||
{
|
||||
static Named_object* builtin_return_address;
|
||||
return this->lower_to_builtin(&builtin_return_address,
|
||||
"__builtin_return_address",
|
||||
0);
|
||||
}
|
||||
else if (n == "getcallersp")
|
||||
else if (this->args_ != NULL
|
||||
&& this->args_->size() == 1
|
||||
&& n == "getcallersp")
|
||||
{
|
||||
// The actual argument to getcallersp is always the
|
||||
// address of a parameter; we don't need that for the
|
||||
// GCC builtin function, so we just ignore it.
|
||||
static Named_object* builtin_frame_address;
|
||||
return this->lower_to_builtin(&builtin_frame_address,
|
||||
"__builtin_frame_address",
|
||||
|
@ -10027,7 +10042,7 @@ Call_expression::do_check_types(Gogo*)
|
|||
}
|
||||
|
||||
const Typed_identifier_list* parameters = fntype->parameters();
|
||||
if (this->args_ == NULL)
|
||||
if (this->args_ == NULL || this->args_->size() == 0)
|
||||
{
|
||||
if (parameters != NULL && !parameters->empty())
|
||||
this->report_error(_("not enough arguments"));
|
||||
|
|
|
@ -91,9 +91,14 @@ DEF_GO_RUNTIME(MAKESLICE64, "runtime.makeslice64", P3(TYPE, INT64, INT64),
|
|||
R1(SLICE))
|
||||
|
||||
|
||||
// Make a map.
|
||||
DEF_GO_RUNTIME(MAKEMAP, "runtime.makemap", P4(TYPE, INT64, POINTER, POINTER),
|
||||
R1(MAP))
|
||||
// Make a map with a hint and an (optional, unused) map structure.
|
||||
DEF_GO_RUNTIME(MAKEMAP, "runtime.makemap", P3(TYPE, INT, POINTER),
|
||||
R1(MAP))
|
||||
DEF_GO_RUNTIME(MAKEMAP64, "runtime.makemap64", P3(TYPE, INT64, POINTER),
|
||||
R1(MAP))
|
||||
|
||||
// Make a map with no hint, or a small constant hint.
|
||||
DEF_GO_RUNTIME(MAKEMAP_SMALL, "runtime.makemap_small", P0(), R1(MAP))
|
||||
|
||||
// Build a map from a composite literal.
|
||||
DEF_GO_RUNTIME(CONSTRUCT_MAP, "__go_construct_map",
|
||||
|
|
|
@ -7830,7 +7830,7 @@ Map_type::do_get_backend(Gogo* gogo)
|
|||
bfields[7].btype = uintptr_type->get_backend(gogo);
|
||||
bfields[7].location = bloc;
|
||||
|
||||
bfields[8].name = "overflow";
|
||||
bfields[8].name = "extra";
|
||||
bfields[8].btype = bpvt;
|
||||
bfields[8].location = bloc;
|
||||
|
||||
|
@ -8144,21 +8144,23 @@ Map_type::hmap_type(Type* bucket_type)
|
|||
|
||||
Type* int_type = Type::lookup_integer_type("int");
|
||||
Type* uint8_type = Type::lookup_integer_type("uint8");
|
||||
Type* uint16_type = Type::lookup_integer_type("uint16");
|
||||
Type* uint32_type = Type::lookup_integer_type("uint32");
|
||||
Type* uintptr_type = Type::lookup_integer_type("uintptr");
|
||||
Type* void_ptr_type = Type::make_pointer_type(Type::make_void_type());
|
||||
|
||||
Type* ptr_bucket_type = Type::make_pointer_type(bucket_type);
|
||||
|
||||
Struct_type* ret = make_builtin_struct_type(8,
|
||||
Struct_type* ret = make_builtin_struct_type(9,
|
||||
"count", int_type,
|
||||
"flags", uint8_type,
|
||||
"B", uint8_type,
|
||||
"noverflow", uint16_type,
|
||||
"hash0", uint32_type,
|
||||
"buckets", ptr_bucket_type,
|
||||
"oldbuckets", ptr_bucket_type,
|
||||
"nevacuate", uintptr_type,
|
||||
"overflow", void_ptr_type);
|
||||
"extra", void_ptr_type);
|
||||
ret->set_is_struct_incomparable();
|
||||
this->hmap_type_ = ret;
|
||||
return ret;
|
||||
|
@ -8191,18 +8193,22 @@ Map_type::hiter_type(Gogo* gogo)
|
|||
Type* hmap_type = this->hmap_type(bucket_type);
|
||||
Type* hmap_ptr_type = Type::make_pointer_type(hmap_type);
|
||||
Type* void_ptr_type = Type::make_pointer_type(Type::make_void_type());
|
||||
Type* bool_type = Type::lookup_bool_type();
|
||||
|
||||
Struct_type* ret = make_builtin_struct_type(12,
|
||||
Struct_type* ret = make_builtin_struct_type(15,
|
||||
"key", key_ptr_type,
|
||||
"val", val_ptr_type,
|
||||
"t", uint8_ptr_type,
|
||||
"h", hmap_ptr_type,
|
||||
"buckets", bucket_ptr_type,
|
||||
"bptr", bucket_ptr_type,
|
||||
"overflow0", void_ptr_type,
|
||||
"overflow1", void_ptr_type,
|
||||
"overflow", void_ptr_type,
|
||||
"oldoverflow", void_ptr_type,
|
||||
"startBucket", uintptr_type,
|
||||
"stuff", uintptr_type,
|
||||
"offset", uint8_type,
|
||||
"wrapped", bool_type,
|
||||
"B", uint8_type,
|
||||
"i", uint8_type,
|
||||
"bucket", uintptr_type,
|
||||
"checkBucket", uintptr_type);
|
||||
ret->set_is_struct_incomparable();
|
||||
|
|
|
@ -2826,6 +2826,9 @@ class Map_type : public Type
|
|||
static Type*
|
||||
make_map_type_descriptor_type();
|
||||
|
||||
// This must be in sync with libgo/go/runtime/hashmap.go.
|
||||
static const int bucket_size = 8;
|
||||
|
||||
protected:
|
||||
int
|
||||
do_traverse(Traverse*);
|
||||
|
@ -2867,7 +2870,6 @@ class Map_type : public Type
|
|||
|
||||
private:
|
||||
// These must be in sync with libgo/go/runtime/hashmap.go.
|
||||
static const int bucket_size = 8;
|
||||
static const int max_key_size = 128;
|
||||
static const int max_val_size = 128;
|
||||
static const int max_zero_size = 1024;
|
||||
|
|
|
@ -331,6 +331,25 @@ Gogo::assign_needs_write_barrier(Expression* lhs)
|
|||
if (!lhs->type()->has_pointer())
|
||||
return false;
|
||||
|
||||
// An assignment to a field is handled like an assignment to the
|
||||
// struct.
|
||||
while (true)
|
||||
{
|
||||
// Nothing to do for a type that can not be in the heap, or a
|
||||
// pointer to a type that can not be in the heap. We check this
|
||||
// at each level of a struct.
|
||||
if (!lhs->type()->in_heap())
|
||||
return false;
|
||||
if (lhs->type()->points_to() != NULL
|
||||
&& !lhs->type()->points_to()->in_heap())
|
||||
return false;
|
||||
|
||||
Field_reference_expression* fre = lhs->field_reference_expression();
|
||||
if (fre == NULL)
|
||||
break;
|
||||
lhs = fre->expr();
|
||||
}
|
||||
|
||||
// Nothing to do for an assignment to a temporary.
|
||||
if (lhs->temporary_reference_expression() != NULL)
|
||||
return false;
|
||||
|
@ -359,12 +378,30 @@ Gogo::assign_needs_write_barrier(Expression* lhs)
|
|||
}
|
||||
}
|
||||
|
||||
// Nothing to do for a type that can not be in the heap, or a
|
||||
// pointer to a type that can not be in the heap.
|
||||
if (!lhs->type()->in_heap())
|
||||
return false;
|
||||
if (lhs->type()->points_to() != NULL && !lhs->type()->points_to()->in_heap())
|
||||
return false;
|
||||
// For a struct assignment, we don't need a write barrier if all the
|
||||
// pointer types can not be in the heap.
|
||||
Struct_type* st = lhs->type()->struct_type();
|
||||
if (st != NULL)
|
||||
{
|
||||
bool in_heap = false;
|
||||
const Struct_field_list* fields = st->fields();
|
||||
for (Struct_field_list::const_iterator p = fields->begin();
|
||||
p != fields->end();
|
||||
p++)
|
||||
{
|
||||
Type* ft = p->type();
|
||||
if (!ft->has_pointer())
|
||||
continue;
|
||||
if (!ft->in_heap())
|
||||
continue;
|
||||
if (ft->points_to() != NULL && !ft->points_to()->in_heap())
|
||||
continue;
|
||||
in_heap = true;
|
||||
break;
|
||||
}
|
||||
if (!in_heap)
|
||||
return false;
|
||||
}
|
||||
|
||||
// Write barrier needed in other cases.
|
||||
return true;
|
||||
|
|
|
@ -1,3 +1,24 @@
|
|||
2018-01-08 Ian Lance Taylor <iant@golang.org>
|
||||
|
||||
* Makefile.am (go_cmd_vet_files): New variable.
|
||||
(go_cmd_buildid_files, go_cmd_test2json_files): New variables.
|
||||
(s-zdefaultcc): Change from constants to functions.
|
||||
(noinst_PROGRAMS): Add vet, buildid, and test2json.
|
||||
(cgo$(EXEEXT)): Link against $(LIBGOTOOL).
|
||||
(vet$(EXEEXT)): New target.
|
||||
(buildid$(EXEEXT)): New target.
|
||||
(test2json$(EXEEXT)): New target.
|
||||
(install-exec-local): Install all $(noinst_PROGRAMS).
|
||||
(uninstall-local): Uninstasll all $(noinst_PROGRAMS).
|
||||
(check-go-tool): Depend on $(noinst_PROGRAMS). Copy down
|
||||
objabi.go.
|
||||
(check-runtime): Depend on $(noinst_PROGRAMS).
|
||||
(check-cgo-test, check-carchive-test): Likewise.
|
||||
(check-vet): New target.
|
||||
(check): Depend on check-vet. Look at cmd_vet-testlog.
|
||||
(.PHONY): Add check-vet.
|
||||
* Makefile.in: Rebuild.
|
||||
|
||||
2017-10-25 Ian Lance Taylor <iant@golang.org>
|
||||
|
||||
* Makefile.am (check-go-tool): Output colon after ${fl}.
|
||||
|
|
|
@ -69,6 +69,40 @@ go_cmd_cgo_files = \
|
|||
$(cmdsrcdir)/cgo/out.go \
|
||||
$(cmdsrcdir)/cgo/util.go
|
||||
|
||||
go_cmd_vet_files = \
|
||||
$(cmdsrcdir)/vet/asmdecl.go \
|
||||
$(cmdsrcdir)/vet/assign.go \
|
||||
$(cmdsrcdir)/vet/atomic.go \
|
||||
$(cmdsrcdir)/vet/bool.go \
|
||||
$(cmdsrcdir)/vet/buildtag.go \
|
||||
$(cmdsrcdir)/vet/cgo.go \
|
||||
$(cmdsrcdir)/vet/composite.go \
|
||||
$(cmdsrcdir)/vet/copylock.go \
|
||||
$(cmdsrcdir)/vet/deadcode.go \
|
||||
$(cmdsrcdir)/vet/dead.go \
|
||||
$(cmdsrcdir)/vet/doc.go \
|
||||
$(cmdsrcdir)/vet/httpresponse.go \
|
||||
$(cmdsrcdir)/vet/lostcancel.go \
|
||||
$(cmdsrcdir)/vet/main.go \
|
||||
$(cmdsrcdir)/vet/method.go \
|
||||
$(cmdsrcdir)/vet/nilfunc.go \
|
||||
$(cmdsrcdir)/vet/print.go \
|
||||
$(cmdsrcdir)/vet/rangeloop.go \
|
||||
$(cmdsrcdir)/vet/shadow.go \
|
||||
$(cmdsrcdir)/vet/shift.go \
|
||||
$(cmdsrcdir)/vet/structtag.go \
|
||||
$(cmdsrcdir)/vet/tests.go \
|
||||
$(cmdsrcdir)/vet/types.go \
|
||||
$(cmdsrcdir)/vet/unsafeptr.go \
|
||||
$(cmdsrcdir)/vet/unused.go
|
||||
|
||||
go_cmd_buildid_files = \
|
||||
$(cmdsrcdir)/buildid/buildid.go \
|
||||
$(cmdsrcdir)/buildid/doc.go
|
||||
|
||||
go_cmd_test2json_files = \
|
||||
$(cmdsrcdir)/test2json/main.go
|
||||
|
||||
GCCGO_INSTALL_NAME := $(shell echo gccgo|sed '$(program_transform_name)')
|
||||
GCC_INSTALL_NAME := $(shell echo gcc|sed '$(program_transform_name)')
|
||||
GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)')
|
||||
|
@ -76,9 +110,9 @@ GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)')
|
|||
zdefaultcc.go: s-zdefaultcc; @true
|
||||
s-zdefaultcc: Makefile
|
||||
echo 'package main' > zdefaultcc.go.tmp
|
||||
echo 'const defaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'const defaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'const defaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'func defaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'func defaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'func defaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'const defaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
|
||||
$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
|
||||
$(STAMP) $@
|
||||
|
@ -97,23 +131,33 @@ if NATIVE
|
|||
# and install them as regular programs.
|
||||
|
||||
bin_PROGRAMS = go$(EXEEXT) gofmt$(EXEEXT)
|
||||
noinst_PROGRAMS = cgo$(EXEEXT)
|
||||
noinst_PROGRAMS = cgo$(EXEEXT) vet$(EXEEXT) buildid$(EXEEXT) test2json$(EXEEXT)
|
||||
man_MANS = go.1 gofmt.1
|
||||
|
||||
go$(EXEEXT): $(go_cmd_go_files) $(LIBGOTOOL) $(LIBGODEP)
|
||||
$(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP)
|
||||
$(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS)
|
||||
cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGODEP)
|
||||
$(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBS) $(NET_LIBS)
|
||||
cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBGODEP)
|
||||
$(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
vet$(EXEEXT): $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBGODEP)
|
||||
$(GOLINK) $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
buildid$(EXEEXT): $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBGODEP)
|
||||
$(GOLINK) $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
test2json$(EXEEXT): $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBGODEP)
|
||||
$(GOLINK) $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
|
||||
install-exec-local: cgo$(EXEEXT)
|
||||
install-exec-local: $(noinst_PROGRAMS)
|
||||
$(MKDIR_P) $(DESTDIR)$(libexecsubdir)
|
||||
rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
|
||||
$(INSTALL_PROGRAM) cgo$(exeext) $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
|
||||
for f in $(noinst_PROGRAMS); do \
|
||||
rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
|
||||
$(INSTALL_PROGRAM) $$f $(DESTDIR)$(libexecsubdir)/$$f; \
|
||||
done
|
||||
|
||||
uninstall-local:
|
||||
rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
|
||||
for f in $(noinst_PROGRAMS); do \
|
||||
rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
|
||||
done
|
||||
|
||||
GOTESTFLAGS =
|
||||
|
||||
|
@ -177,8 +221,8 @@ CHECK_ENV = \
|
|||
# It assumes that abs_libgodir is set.
|
||||
ECHO_ENV = PATH=`echo $(abs_builddir):$${PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'` GCCGO='$(abs_builddir)/check-gccgo' CC='$(abs_builddir)/check-gcc' GCCGOTOOLDIR='$(abs_builddir)' GO_TESTING_GOTOOLS=yes LD_LIBRARY_PATH=`echo $${abs_libgodir}/.libs:$${LD_LIBRARY_PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'` GOROOT=`echo $${abs_libgodir}`
|
||||
|
||||
# check-go-tools runs `go test cmd/go` in our environment.
|
||||
check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
||||
# check-go-tool runs `go test cmd/go` in our environment.
|
||||
check-go-tool: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
rm -rf check-go-dir cmd_go-testlog
|
||||
$(MKDIR_P) check-go-dir/src/cmd/go
|
||||
cp $(cmdsrcdir)/go/*.go check-go-dir/src/cmd/go/
|
||||
|
@ -187,6 +231,7 @@ check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
|||
cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/
|
||||
cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/
|
||||
cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/
|
||||
cp $(libgodir)/objabi.go check-go-dir/src/cmd/internal/objabi/
|
||||
@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
|
||||
abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \
|
||||
echo "cd check-go-dir/src/cmd/go && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_go-testlog
|
||||
|
@ -200,7 +245,7 @@ check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
|||
# The runtime package is also tested as part of libgo,
|
||||
# but the runtime tests use the go tool heavily, so testing
|
||||
# here too will catch more problems.
|
||||
check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
||||
check-runtime: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
rm -rf check-runtime-dir runtime-testlog
|
||||
$(MKDIR_P) check-runtime-dir
|
||||
@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
|
||||
|
@ -219,7 +264,7 @@ check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
|||
grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
|
||||
|
||||
# check-cgo-test runs `go test misc/cgo/test` in our environment.
|
||||
check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
||||
check-cgo-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
rm -rf cgo-test-dir cgo-testlog
|
||||
$(MKDIR_P) cgo-test-dir/misc/cgo
|
||||
cp -r $(libgomiscdir)/cgo/test cgo-test-dir/misc/cgo/
|
||||
|
@ -233,7 +278,7 @@ check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
|||
|
||||
# check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go`
|
||||
# in our environment.
|
||||
check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
||||
check-carchive-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
rm -rf carchive-test-dir carchive-testlog
|
||||
$(MKDIR_P) carchive-test-dir/misc/cgo
|
||||
cp -r $(libgomiscdir)/cgo/testcarchive carchive-test-dir/misc/cgo/
|
||||
|
@ -245,11 +290,25 @@ check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
|||
(cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
|
||||
grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
|
||||
|
||||
# check-vet runs `go test cmd/vet` in our environment.
|
||||
check-vet: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
rm -rf check-vet-dir cmd_vet-testlog
|
||||
$(MKDIR_P) check-vet-dir/src/cmd
|
||||
cp -r $(cmdsrcdir)/vet check-vet-dir/src/cmd/
|
||||
@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
|
||||
abs_checkdir=`cd check-vet-dir && $(PWD_COMMAND)`; \
|
||||
echo "cd check-vet-dir/src/cmd/vet && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_vet-testlog
|
||||
$(CHECK_ENV) \
|
||||
GOPATH=`cd check-vet-dir && $(PWD_COMMAND)`; \
|
||||
export GOPATH; \
|
||||
(cd check-vet-dir/src/cmd/vet && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_vet-testlog 2>&1 || echo "--- $${fl}: go test cmd/vet (0.00s)" >> cmd_vet-testlog
|
||||
grep '^--- ' cmd_vet-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
|
||||
|
||||
# The check targets runs the tests and assembles the output files.
|
||||
check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test
|
||||
check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
|
||||
@mv gotools.head gotools.sum
|
||||
@cp gotools.sum gotools.log
|
||||
@for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog; do \
|
||||
@for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog cmd_vet-testlog; do \
|
||||
testname=`echo $${file} | sed -e 's/-testlog//' -e 's|_|/|'`; \
|
||||
echo "Running $${testname}" >> gotools.sum; \
|
||||
echo "Running $${testname}" >> gotools.log; \
|
||||
|
@ -275,7 +334,7 @@ check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test
|
|||
@echo "runtest completed at `date`" >> gotools.log
|
||||
@if grep '^FAIL' gotools.sum >/dev/null 2>&1; then exit 1; fi
|
||||
|
||||
.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test
|
||||
.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
|
||||
|
||||
else
|
||||
|
||||
|
|
|
@ -88,6 +88,9 @@ CONFIG_CLEAN_FILES =
|
|||
CONFIG_CLEAN_VPATH_FILES =
|
||||
am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)"
|
||||
PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS)
|
||||
buildid_SOURCES = buildid.c
|
||||
buildid_OBJECTS = buildid.$(OBJEXT)
|
||||
buildid_LDADD = $(LDADD)
|
||||
cgo_SOURCES = cgo.c
|
||||
cgo_OBJECTS = cgo.$(OBJEXT)
|
||||
cgo_LDADD = $(LDADD)
|
||||
|
@ -97,6 +100,12 @@ go_LDADD = $(LDADD)
|
|||
gofmt_SOURCES = gofmt.c
|
||||
gofmt_OBJECTS = gofmt.$(OBJEXT)
|
||||
gofmt_LDADD = $(LDADD)
|
||||
test2json_SOURCES = test2json.c
|
||||
test2json_OBJECTS = test2json.$(OBJEXT)
|
||||
test2json_LDADD = $(LDADD)
|
||||
vet_SOURCES = vet.c
|
||||
vet_OBJECTS = vet.$(OBJEXT)
|
||||
vet_LDADD = $(LDADD)
|
||||
DEFAULT_INCLUDES = -I.@am__isrc@
|
||||
depcomp = $(SHELL) $(top_srcdir)/../depcomp
|
||||
am__depfiles_maybe = depfiles
|
||||
|
@ -105,7 +114,7 @@ COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
|
|||
$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
|
||||
CCLD = $(CC)
|
||||
LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
|
||||
SOURCES = cgo.c go.c gofmt.c
|
||||
SOURCES = buildid.c cgo.c go.c gofmt.c test2json.c vet.c
|
||||
am__can_run_installinfo = \
|
||||
case $$AM_UPDATE_INFO_DIR in \
|
||||
n|no|NO) false;; \
|
||||
|
@ -288,6 +297,40 @@ go_cmd_cgo_files = \
|
|||
$(cmdsrcdir)/cgo/out.go \
|
||||
$(cmdsrcdir)/cgo/util.go
|
||||
|
||||
go_cmd_vet_files = \
|
||||
$(cmdsrcdir)/vet/asmdecl.go \
|
||||
$(cmdsrcdir)/vet/assign.go \
|
||||
$(cmdsrcdir)/vet/atomic.go \
|
||||
$(cmdsrcdir)/vet/bool.go \
|
||||
$(cmdsrcdir)/vet/buildtag.go \
|
||||
$(cmdsrcdir)/vet/cgo.go \
|
||||
$(cmdsrcdir)/vet/composite.go \
|
||||
$(cmdsrcdir)/vet/copylock.go \
|
||||
$(cmdsrcdir)/vet/deadcode.go \
|
||||
$(cmdsrcdir)/vet/dead.go \
|
||||
$(cmdsrcdir)/vet/doc.go \
|
||||
$(cmdsrcdir)/vet/httpresponse.go \
|
||||
$(cmdsrcdir)/vet/lostcancel.go \
|
||||
$(cmdsrcdir)/vet/main.go \
|
||||
$(cmdsrcdir)/vet/method.go \
|
||||
$(cmdsrcdir)/vet/nilfunc.go \
|
||||
$(cmdsrcdir)/vet/print.go \
|
||||
$(cmdsrcdir)/vet/rangeloop.go \
|
||||
$(cmdsrcdir)/vet/shadow.go \
|
||||
$(cmdsrcdir)/vet/shift.go \
|
||||
$(cmdsrcdir)/vet/structtag.go \
|
||||
$(cmdsrcdir)/vet/tests.go \
|
||||
$(cmdsrcdir)/vet/types.go \
|
||||
$(cmdsrcdir)/vet/unsafeptr.go \
|
||||
$(cmdsrcdir)/vet/unused.go
|
||||
|
||||
go_cmd_buildid_files = \
|
||||
$(cmdsrcdir)/buildid/buildid.go \
|
||||
$(cmdsrcdir)/buildid/doc.go
|
||||
|
||||
go_cmd_test2json_files = \
|
||||
$(cmdsrcdir)/test2json/main.go
|
||||
|
||||
GCCGO_INSTALL_NAME := $(shell echo gccgo|sed '$(program_transform_name)')
|
||||
GCC_INSTALL_NAME := $(shell echo gcc|sed '$(program_transform_name)')
|
||||
GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)')
|
||||
|
@ -300,7 +343,7 @@ MOSTLYCLEANFILES = \
|
|||
# For a native build we build the programs using the newly built libgo
|
||||
# and install them as regular programs.
|
||||
@NATIVE_TRUE@bin_PROGRAMS = go$(EXEEXT) gofmt$(EXEEXT)
|
||||
@NATIVE_TRUE@noinst_PROGRAMS = cgo$(EXEEXT)
|
||||
@NATIVE_TRUE@noinst_PROGRAMS = cgo$(EXEEXT) vet$(EXEEXT) buildid$(EXEEXT) test2json$(EXEEXT)
|
||||
@NATIVE_TRUE@man_MANS = go.1 gofmt.1
|
||||
@NATIVE_TRUE@GOTESTFLAGS =
|
||||
|
||||
|
@ -411,6 +454,9 @@ clean-binPROGRAMS:
|
|||
|
||||
clean-noinstPROGRAMS:
|
||||
-test -z "$(noinst_PROGRAMS)" || rm -f $(noinst_PROGRAMS)
|
||||
@NATIVE_FALSE@buildid$(EXEEXT): $(buildid_OBJECTS) $(buildid_DEPENDENCIES) $(EXTRA_buildid_DEPENDENCIES)
|
||||
@NATIVE_FALSE@ @rm -f buildid$(EXEEXT)
|
||||
@NATIVE_FALSE@ $(LINK) $(buildid_OBJECTS) $(buildid_LDADD) $(LIBS)
|
||||
@NATIVE_FALSE@cgo$(EXEEXT): $(cgo_OBJECTS) $(cgo_DEPENDENCIES) $(EXTRA_cgo_DEPENDENCIES)
|
||||
@NATIVE_FALSE@ @rm -f cgo$(EXEEXT)
|
||||
@NATIVE_FALSE@ $(LINK) $(cgo_OBJECTS) $(cgo_LDADD) $(LIBS)
|
||||
|
@ -420,6 +466,12 @@ clean-noinstPROGRAMS:
|
|||
@NATIVE_FALSE@gofmt$(EXEEXT): $(gofmt_OBJECTS) $(gofmt_DEPENDENCIES) $(EXTRA_gofmt_DEPENDENCIES)
|
||||
@NATIVE_FALSE@ @rm -f gofmt$(EXEEXT)
|
||||
@NATIVE_FALSE@ $(LINK) $(gofmt_OBJECTS) $(gofmt_LDADD) $(LIBS)
|
||||
@NATIVE_FALSE@test2json$(EXEEXT): $(test2json_OBJECTS) $(test2json_DEPENDENCIES) $(EXTRA_test2json_DEPENDENCIES)
|
||||
@NATIVE_FALSE@ @rm -f test2json$(EXEEXT)
|
||||
@NATIVE_FALSE@ $(LINK) $(test2json_OBJECTS) $(test2json_LDADD) $(LIBS)
|
||||
@NATIVE_FALSE@vet$(EXEEXT): $(vet_OBJECTS) $(vet_DEPENDENCIES) $(EXTRA_vet_DEPENDENCIES)
|
||||
@NATIVE_FALSE@ @rm -f vet$(EXEEXT)
|
||||
@NATIVE_FALSE@ $(LINK) $(vet_OBJECTS) $(vet_LDADD) $(LIBS)
|
||||
|
||||
mostlyclean-compile:
|
||||
-rm -f *.$(OBJEXT)
|
||||
|
@ -427,9 +479,12 @@ mostlyclean-compile:
|
|||
distclean-compile:
|
||||
-rm -f *.tab.c
|
||||
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/buildid.Po@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cgo.Po@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/go.Po@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gofmt.Po@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test2json.Po@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/vet.Po@am__quote@
|
||||
|
||||
.c.o:
|
||||
@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
|
||||
|
@ -676,9 +731,9 @@ uninstall-man: uninstall-man1
|
|||
zdefaultcc.go: s-zdefaultcc; @true
|
||||
s-zdefaultcc: Makefile
|
||||
echo 'package main' > zdefaultcc.go.tmp
|
||||
echo 'const defaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'const defaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'const defaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'func defaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'func defaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'func defaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'const defaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
|
||||
$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
|
||||
$(STAMP) $@
|
||||
|
@ -690,16 +745,26 @@ mostlyclean-local:
|
|||
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
@NATIVE_TRUE@gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP)
|
||||
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS)
|
||||
@NATIVE_TRUE@cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGODEP)
|
||||
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBS) $(NET_LIBS)
|
||||
@NATIVE_TRUE@cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBGODEP)
|
||||
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
@NATIVE_TRUE@vet$(EXEEXT): $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBGODEP)
|
||||
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
@NATIVE_TRUE@buildid$(EXEEXT): $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBGODEP)
|
||||
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
@NATIVE_TRUE@test2json$(EXEEXT): $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBGODEP)
|
||||
@NATIVE_TRUE@ $(GOLINK) $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
|
||||
|
||||
@NATIVE_TRUE@install-exec-local: cgo$(EXEEXT)
|
||||
@NATIVE_TRUE@install-exec-local: $(noinst_PROGRAMS)
|
||||
@NATIVE_TRUE@ $(MKDIR_P) $(DESTDIR)$(libexecsubdir)
|
||||
@NATIVE_TRUE@ rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
|
||||
@NATIVE_TRUE@ $(INSTALL_PROGRAM) cgo$(exeext) $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
|
||||
@NATIVE_TRUE@ for f in $(noinst_PROGRAMS); do \
|
||||
@NATIVE_TRUE@ rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
|
||||
@NATIVE_TRUE@ $(INSTALL_PROGRAM) $$f $(DESTDIR)$(libexecsubdir)/$$f; \
|
||||
@NATIVE_TRUE@ done
|
||||
|
||||
@NATIVE_TRUE@uninstall-local:
|
||||
@NATIVE_TRUE@ rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
|
||||
@NATIVE_TRUE@ for f in $(noinst_PROGRAMS); do \
|
||||
@NATIVE_TRUE@ rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
|
||||
@NATIVE_TRUE@ done
|
||||
|
||||
# Run tests using the go tool, and frob the output to look like that
|
||||
# generated by DejaGNU. The main output of this is two files:
|
||||
|
@ -735,8 +800,8 @@ mostlyclean-local:
|
|||
@NATIVE_TRUE@ chmod +x $@.tmp
|
||||
@NATIVE_TRUE@ mv -f $@.tmp $@
|
||||
|
||||
# check-go-tools runs `go test cmd/go` in our environment.
|
||||
@NATIVE_TRUE@check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
||||
# check-go-tool runs `go test cmd/go` in our environment.
|
||||
@NATIVE_TRUE@check-go-tool: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
@NATIVE_TRUE@ rm -rf check-go-dir cmd_go-testlog
|
||||
@NATIVE_TRUE@ $(MKDIR_P) check-go-dir/src/cmd/go
|
||||
@NATIVE_TRUE@ cp $(cmdsrcdir)/go/*.go check-go-dir/src/cmd/go/
|
||||
|
@ -745,6 +810,7 @@ mostlyclean-local:
|
|||
@NATIVE_TRUE@ cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/
|
||||
@NATIVE_TRUE@ cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/
|
||||
@NATIVE_TRUE@ cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/
|
||||
@NATIVE_TRUE@ cp $(libgodir)/objabi.go check-go-dir/src/cmd/internal/objabi/
|
||||
@NATIVE_TRUE@ @abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
|
||||
@NATIVE_TRUE@ abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \
|
||||
@NATIVE_TRUE@ echo "cd check-go-dir/src/cmd/go && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_go-testlog
|
||||
|
@ -758,7 +824,7 @@ mostlyclean-local:
|
|||
# The runtime package is also tested as part of libgo,
|
||||
# but the runtime tests use the go tool heavily, so testing
|
||||
# here too will catch more problems.
|
||||
@NATIVE_TRUE@check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
||||
@NATIVE_TRUE@check-runtime: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
@NATIVE_TRUE@ rm -rf check-runtime-dir runtime-testlog
|
||||
@NATIVE_TRUE@ $(MKDIR_P) check-runtime-dir
|
||||
@NATIVE_TRUE@ @abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
|
||||
|
@ -777,7 +843,7 @@ mostlyclean-local:
|
|||
@NATIVE_TRUE@ grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
|
||||
|
||||
# check-cgo-test runs `go test misc/cgo/test` in our environment.
|
||||
@NATIVE_TRUE@check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
||||
@NATIVE_TRUE@check-cgo-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
@NATIVE_TRUE@ rm -rf cgo-test-dir cgo-testlog
|
||||
@NATIVE_TRUE@ $(MKDIR_P) cgo-test-dir/misc/cgo
|
||||
@NATIVE_TRUE@ cp -r $(libgomiscdir)/cgo/test cgo-test-dir/misc/cgo/
|
||||
|
@ -791,7 +857,7 @@ mostlyclean-local:
|
|||
|
||||
# check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go`
|
||||
# in our environment.
|
||||
@NATIVE_TRUE@check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
|
||||
@NATIVE_TRUE@check-carchive-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
@NATIVE_TRUE@ rm -rf carchive-test-dir carchive-testlog
|
||||
@NATIVE_TRUE@ $(MKDIR_P) carchive-test-dir/misc/cgo
|
||||
@NATIVE_TRUE@ cp -r $(libgomiscdir)/cgo/testcarchive carchive-test-dir/misc/cgo/
|
||||
|
@ -803,11 +869,25 @@ mostlyclean-local:
|
|||
@NATIVE_TRUE@ (cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
|
||||
@NATIVE_TRUE@ grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
|
||||
|
||||
# check-vet runs `go test cmd/vet` in our environment.
|
||||
@NATIVE_TRUE@check-vet: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
|
||||
@NATIVE_TRUE@ rm -rf check-vet-dir cmd_vet-testlog
|
||||
@NATIVE_TRUE@ $(MKDIR_P) check-vet-dir/src/cmd
|
||||
@NATIVE_TRUE@ cp -r $(cmdsrcdir)/vet check-vet-dir/src/cmd/
|
||||
@NATIVE_TRUE@ @abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
|
||||
@NATIVE_TRUE@ abs_checkdir=`cd check-vet-dir && $(PWD_COMMAND)`; \
|
||||
@NATIVE_TRUE@ echo "cd check-vet-dir/src/cmd/vet && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_vet-testlog
|
||||
@NATIVE_TRUE@ $(CHECK_ENV) \
|
||||
@NATIVE_TRUE@ GOPATH=`cd check-vet-dir && $(PWD_COMMAND)`; \
|
||||
@NATIVE_TRUE@ export GOPATH; \
|
||||
@NATIVE_TRUE@ (cd check-vet-dir/src/cmd/vet && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_vet-testlog 2>&1 || echo "--- $${fl}: go test cmd/vet (0.00s)" >> cmd_vet-testlog
|
||||
@NATIVE_TRUE@ grep '^--- ' cmd_vet-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
|
||||
|
||||
# The check targets runs the tests and assembles the output files.
|
||||
@NATIVE_TRUE@check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test
|
||||
@NATIVE_TRUE@check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
|
||||
@NATIVE_TRUE@ @mv gotools.head gotools.sum
|
||||
@NATIVE_TRUE@ @cp gotools.sum gotools.log
|
||||
@NATIVE_TRUE@ @for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog; do \
|
||||
@NATIVE_TRUE@ @for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog cmd_vet-testlog; do \
|
||||
@NATIVE_TRUE@ testname=`echo $${file} | sed -e 's/-testlog//' -e 's|_|/|'`; \
|
||||
@NATIVE_TRUE@ echo "Running $${testname}" >> gotools.sum; \
|
||||
@NATIVE_TRUE@ echo "Running $${testname}" >> gotools.log; \
|
||||
|
@ -833,7 +913,7 @@ mostlyclean-local:
|
|||
@NATIVE_TRUE@ @echo "runtest completed at `date`" >> gotools.log
|
||||
@NATIVE_TRUE@ @if grep '^FAIL' gotools.sum >/dev/null 2>&1; then exit 1; fi
|
||||
|
||||
@NATIVE_TRUE@.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test
|
||||
@NATIVE_TRUE@.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
|
||||
|
||||
# For a non-native build we have to build the programs using a
|
||||
# previously built host (or build -> host) Go compiler. We should
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
c8aec4095e089ff6ac50d18e97c3f46561f14f48
|
||||
9ce6b5c2ed5d3d5251b9a6a0c548d5fb2c8567e8
|
||||
|
||||
The first line of this file holds the git revision number of the
|
||||
last merge done from the master library sources.
|
||||
|
|
|
@ -400,8 +400,11 @@ toolexeclibgounicode_DATA = \
|
|||
# internal packages nothing will explicitly depend on them.
|
||||
# Force them to be built.
|
||||
noinst_DATA = \
|
||||
golang_org/x/net/internal/nettest.gox \
|
||||
golang_org/x/net/nettest.gox \
|
||||
internal/testenv.gox \
|
||||
net/internal/socktest.gox
|
||||
net/internal/socktest.gox \
|
||||
os/signal/internal/pty.gox
|
||||
|
||||
if LIBGO_IS_RTEMS
|
||||
rtems_task_variable_add_file = runtime/rtems-task-variable-add.c
|
||||
|
@ -533,6 +536,24 @@ s-version: Makefile
|
|||
$(SHELL) $(srcdir)/mvifdiff.sh version.go.tmp version.go
|
||||
$(STAMP) $@
|
||||
|
||||
objabi.go: s-objabi; @true
|
||||
s-objabi: Makefile
|
||||
rm -f objabi.go.tmp
|
||||
echo "package objabi" > objabi.go.tmp
|
||||
echo "import \"runtime\"" >> objabi.go.tmp
|
||||
echo 'const defaultGOROOT = `$(prefix)`' >> objabi.go.tmp
|
||||
echo 'const defaultGO386 = `sse2`' >> objabi.go.tmp
|
||||
echo 'const defaultGOARM = `5`' >> objabi.go.tmp
|
||||
echo 'const defaultGOMIPS = `hardfloat`' >> objabi.go.tmp
|
||||
echo 'const defaultGOOS = runtime.GOOS' >> objabi.go.tmp
|
||||
echo 'const defaultGOARCH = runtime.GOARCH' >> objabi.go.tmp
|
||||
echo 'const defaultGO_EXTLINK_ENABLED = ``' >> objabi.go.tmp
|
||||
echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> objabi.go.tmp
|
||||
echo 'const stackGuardMultiplier = 1' >> objabi.go.tmp
|
||||
echo 'const goexperiment = ``' >> objabi.go.tmp
|
||||
$(SHELL) $(srcdir)/mvifdiff.sh objabi.go.tmp objabi.go
|
||||
$(STAMP) $@
|
||||
|
||||
runtime_sysinfo.go: s-runtime_sysinfo; @true
|
||||
s-runtime_sysinfo: $(srcdir)/mkrsysinfo.sh gen-sysinfo.go
|
||||
GOARCH=$(GOARCH) GOOS=$(GOOS) $(SHELL) $(srcdir)/mkrsysinfo.sh
|
||||
|
@ -553,10 +574,11 @@ zdefaultcc.go: s-zdefaultcc; @true
|
|||
s-zdefaultcc: Makefile
|
||||
echo 'package cfg' > zdefaultcc.go.tmp
|
||||
echo >> zdefaultcc.go.tmp
|
||||
echo 'const DefaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'const DefaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'const DefaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'func DefaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'func DefaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'func DefaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'const DefaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
|
||||
echo 'var OSArchSupportsCgo = map[string]bool{}' >> zdefaultcc.go.tmp
|
||||
$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
|
||||
$(STAMP) $@
|
||||
|
||||
|
@ -758,11 +780,15 @@ PACKAGES = \
|
|||
go/types \
|
||||
golang_org/x/crypto/chacha20poly1305 \
|
||||
golang_org/x/crypto/chacha20poly1305/internal/chacha20 \
|
||||
golang_org/x/crypto/cryptobyte \
|
||||
golang_org/x/crypto/cryptobyte/asn1 \
|
||||
golang_org/x/crypto/curve25519 \
|
||||
golang_org/x/crypto/poly1305 \
|
||||
golang_org/x/net/http2/hpack \
|
||||
golang_org/x/net/idna \
|
||||
golang_org/x/net/internal/nettest \
|
||||
golang_org/x/net/lex/httplex \
|
||||
golang_org/x/net/nettest \
|
||||
golang_org/x/net/proxy \
|
||||
golang_org/x/text/secure/bidirule \
|
||||
golang_org/x/text/transform \
|
||||
|
@ -824,6 +850,7 @@ PACKAGES = \
|
|||
os \
|
||||
os/exec \
|
||||
os/signal \
|
||||
os/signal/internal/pty \
|
||||
os/user \
|
||||
path \
|
||||
path/filepath \
|
||||
|
@ -905,7 +932,7 @@ libgolibbegin_a_CFLAGS = $(AM_CFLAGS) -fPIC
|
|||
GOTOOL_PACKAGES = \
|
||||
cmd/go/internal/base \
|
||||
cmd/go/internal/bug \
|
||||
cmd/go/internal/buildid \
|
||||
cmd/go/internal/cache \
|
||||
cmd/go/internal/cfg \
|
||||
cmd/go/internal/clean \
|
||||
cmd/go/internal/cmdflag \
|
||||
|
@ -927,7 +954,12 @@ GOTOOL_PACKAGES = \
|
|||
cmd/go/internal/web \
|
||||
cmd/go/internal/work \
|
||||
cmd/internal/browser \
|
||||
cmd/internal/objabi
|
||||
cmd/internal/buildid \
|
||||
cmd/internal/edit \
|
||||
cmd/internal/objabi \
|
||||
cmd/internal/test2json \
|
||||
cmd/vet/internal/cfg \
|
||||
cmd/vet/internal/whitelist
|
||||
|
||||
libgotool_a_SOURCES =
|
||||
libgotool_a_DEPENDENCIES = $(addsuffix .lo,$(GOTOOL_PACKAGES))
|
||||
|
@ -1136,17 +1168,23 @@ runtime_pprof_check_GOCFLAGS = -static-libgo -fno-inline
|
|||
extra_go_files_runtime_internal_sys = version.go
|
||||
runtime/internal/sys.lo.dep: $(extra_go_files_runtime_internal_sys)
|
||||
|
||||
extra_go_files_cmd_internal_objabi = objabi.go
|
||||
cmd/internal/objabi.lo.dep: $(extra_go_files_cmd_internal_objabi)
|
||||
|
||||
extra_go_files_cmd_go_internal_cfg = zdefaultcc.go
|
||||
cmd/go/internal/cfg.lo.dep: $(extra_go_files_cmd_go_internal_cfg)
|
||||
|
||||
extra_go_files_cmd_go_internal_load = zstdpkglist.go
|
||||
cmd/go/internal/load.lo.dep: $(extra_go_files_cmd_go_internal_load)
|
||||
|
||||
extra_check_libs_cmd_go_internal_cache = $(abs_builddir)/libgotool.a
|
||||
extra_check_libs_cmd_go_internal_generate = $(abs_builddir)/libgotool.a
|
||||
extra_check_libs_cmd_go_internal_get = $(abs_builddir)/libgotool.a
|
||||
extra_check_libs_cmd_go_internal_load = $(abs_builddir)/libgotool.a
|
||||
extra_check_libs_cmd_go_internal_work = $(abs_builddir)/libgotool.a
|
||||
|
||||
extra_check_libs_cmd_vet_internal_cfg = $(abs_builddir)/libgotool.a
|
||||
|
||||
# FIXME: The following C files may as well move to the runtime
|
||||
# directory and be treated like other C files.
|
||||
|
||||
|
@ -1233,10 +1271,12 @@ TEST_PACKAGES = \
|
|||
bufio/check \
|
||||
bytes/check \
|
||||
context/check \
|
||||
crypto/check \
|
||||
errors/check \
|
||||
expvar/check \
|
||||
flag/check \
|
||||
fmt/check \
|
||||
hash/check \
|
||||
html/check \
|
||||
image/check \
|
||||
io/check \
|
||||
|
@ -1258,11 +1298,16 @@ TEST_PACKAGES = \
|
|||
unicode/check \
|
||||
archive/tar/check \
|
||||
archive/zip/check \
|
||||
cmd/go/internal/cache/check \
|
||||
cmd/go/internal/generate/check \
|
||||
cmd/go/internal/get/check \
|
||||
cmd/go/internal/load/check \
|
||||
cmd/go/internal/work/check \
|
||||
cmd/internal/buildid/check \
|
||||
cmd/internal/edit/check \
|
||||
cmd/internal/objabi/check \
|
||||
cmd/internal/test2json/check \
|
||||
cmd/vet/internal/cfg/check \
|
||||
compress/bzip2/check \
|
||||
compress/flate/check \
|
||||
compress/gzip/check \
|
||||
|
@ -1315,6 +1360,7 @@ TEST_PACKAGES = \
|
|||
go/constant/check \
|
||||
go/doc/check \
|
||||
go/format/check \
|
||||
go/importer/check \
|
||||
go/internal/gcimporter/check \
|
||||
go/internal/gccgoimporter/check \
|
||||
go/internal/srcimporter/check \
|
||||
|
@ -1325,6 +1371,7 @@ TEST_PACKAGES = \
|
|||
go/types/check \
|
||||
golang_org/x/crypto/chacha20poly1305/check \
|
||||
golang_org/x/crypto/chacha20poly1305/internal/chacha20/check \
|
||||
golang_org/x/crypto/cryptobyte/check \
|
||||
golang_org/x/crypto/curve25519/check \
|
||||
golang_org/x/crypto/poly1305/check \
|
||||
golang_org/x/net/http2/hpack/check \
|
||||
|
|
|
@ -770,7 +770,9 @@ toolexeclibgounicode_DATA = \
|
|||
# Some packages are only needed for tests, so unlike the other
|
||||
# internal packages nothing will explicitly depend on them.
|
||||
# Force them to be built.
|
||||
noinst_DATA = internal/testenv.gox net/internal/socktest.gox \
|
||||
noinst_DATA = golang_org/x/net/internal/nettest.gox \
|
||||
golang_org/x/net/nettest.gox internal/testenv.gox \
|
||||
net/internal/socktest.gox os/signal/internal/pty.gox \
|
||||
zstdpkglist.go zdefaultcc.go
|
||||
@LIBGO_IS_RTEMS_FALSE@rtems_task_variable_add_file =
|
||||
@LIBGO_IS_RTEMS_TRUE@rtems_task_variable_add_file = runtime/rtems-task-variable-add.c
|
||||
|
@ -909,11 +911,15 @@ PACKAGES = \
|
|||
go/types \
|
||||
golang_org/x/crypto/chacha20poly1305 \
|
||||
golang_org/x/crypto/chacha20poly1305/internal/chacha20 \
|
||||
golang_org/x/crypto/cryptobyte \
|
||||
golang_org/x/crypto/cryptobyte/asn1 \
|
||||
golang_org/x/crypto/curve25519 \
|
||||
golang_org/x/crypto/poly1305 \
|
||||
golang_org/x/net/http2/hpack \
|
||||
golang_org/x/net/idna \
|
||||
golang_org/x/net/internal/nettest \
|
||||
golang_org/x/net/lex/httplex \
|
||||
golang_org/x/net/nettest \
|
||||
golang_org/x/net/proxy \
|
||||
golang_org/x/text/secure/bidirule \
|
||||
golang_org/x/text/transform \
|
||||
|
@ -975,6 +981,7 @@ PACKAGES = \
|
|||
os \
|
||||
os/exec \
|
||||
os/signal \
|
||||
os/signal/internal/pty \
|
||||
os/user \
|
||||
path \
|
||||
path/filepath \
|
||||
|
@ -1053,7 +1060,7 @@ libgolibbegin_a_CFLAGS = $(AM_CFLAGS) -fPIC
|
|||
GOTOOL_PACKAGES = \
|
||||
cmd/go/internal/base \
|
||||
cmd/go/internal/bug \
|
||||
cmd/go/internal/buildid \
|
||||
cmd/go/internal/cache \
|
||||
cmd/go/internal/cfg \
|
||||
cmd/go/internal/clean \
|
||||
cmd/go/internal/cmdflag \
|
||||
|
@ -1075,7 +1082,12 @@ GOTOOL_PACKAGES = \
|
|||
cmd/go/internal/web \
|
||||
cmd/go/internal/work \
|
||||
cmd/internal/browser \
|
||||
cmd/internal/objabi
|
||||
cmd/internal/buildid \
|
||||
cmd/internal/edit \
|
||||
cmd/internal/objabi \
|
||||
cmd/internal/test2json \
|
||||
cmd/vet/internal/cfg \
|
||||
cmd/vet/internal/whitelist
|
||||
|
||||
libgotool_a_SOURCES =
|
||||
libgotool_a_DEPENDENCIES = $(addsuffix .lo,$(GOTOOL_PACKAGES))
|
||||
|
@ -1210,12 +1222,15 @@ runtime_internal_sys_lo_check_GOCFLAGS = -fgo-compiling-runtime
|
|||
# Also use -fno-inline to get better results from the memory profiler.
|
||||
runtime_pprof_check_GOCFLAGS = -static-libgo -fno-inline
|
||||
extra_go_files_runtime_internal_sys = version.go
|
||||
extra_go_files_cmd_internal_objabi = objabi.go
|
||||
extra_go_files_cmd_go_internal_cfg = zdefaultcc.go
|
||||
extra_go_files_cmd_go_internal_load = zstdpkglist.go
|
||||
extra_check_libs_cmd_go_internal_cache = $(abs_builddir)/libgotool.a
|
||||
extra_check_libs_cmd_go_internal_generate = $(abs_builddir)/libgotool.a
|
||||
extra_check_libs_cmd_go_internal_get = $(abs_builddir)/libgotool.a
|
||||
extra_check_libs_cmd_go_internal_load = $(abs_builddir)/libgotool.a
|
||||
extra_check_libs_cmd_go_internal_work = $(abs_builddir)/libgotool.a
|
||||
extra_check_libs_cmd_vet_internal_cfg = $(abs_builddir)/libgotool.a
|
||||
@HAVE_STAT_TIMESPEC_FALSE@@LIBGO_IS_SOLARIS_TRUE@matchargs_os =
|
||||
|
||||
# Solaris 11.4 changed the type of fields in struct stat.
|
||||
|
@ -1238,10 +1253,12 @@ TEST_PACKAGES = \
|
|||
bufio/check \
|
||||
bytes/check \
|
||||
context/check \
|
||||
crypto/check \
|
||||
errors/check \
|
||||
expvar/check \
|
||||
flag/check \
|
||||
fmt/check \
|
||||
hash/check \
|
||||
html/check \
|
||||
image/check \
|
||||
io/check \
|
||||
|
@ -1263,11 +1280,16 @@ TEST_PACKAGES = \
|
|||
unicode/check \
|
||||
archive/tar/check \
|
||||
archive/zip/check \
|
||||
cmd/go/internal/cache/check \
|
||||
cmd/go/internal/generate/check \
|
||||
cmd/go/internal/get/check \
|
||||
cmd/go/internal/load/check \
|
||||
cmd/go/internal/work/check \
|
||||
cmd/internal/buildid/check \
|
||||
cmd/internal/edit/check \
|
||||
cmd/internal/objabi/check \
|
||||
cmd/internal/test2json/check \
|
||||
cmd/vet/internal/cfg/check \
|
||||
compress/bzip2/check \
|
||||
compress/flate/check \
|
||||
compress/gzip/check \
|
||||
|
@ -1320,6 +1342,7 @@ TEST_PACKAGES = \
|
|||
go/constant/check \
|
||||
go/doc/check \
|
||||
go/format/check \
|
||||
go/importer/check \
|
||||
go/internal/gcimporter/check \
|
||||
go/internal/gccgoimporter/check \
|
||||
go/internal/srcimporter/check \
|
||||
|
@ -1330,6 +1353,7 @@ TEST_PACKAGES = \
|
|||
go/types/check \
|
||||
golang_org/x/crypto/chacha20poly1305/check \
|
||||
golang_org/x/crypto/chacha20poly1305/internal/chacha20/check \
|
||||
golang_org/x/crypto/cryptobyte/check \
|
||||
golang_org/x/crypto/curve25519/check \
|
||||
golang_org/x/crypto/poly1305/check \
|
||||
golang_org/x/net/http2/hpack/check \
|
||||
|
@ -3130,6 +3154,24 @@ s-version: Makefile
|
|||
$(SHELL) $(srcdir)/mvifdiff.sh version.go.tmp version.go
|
||||
$(STAMP) $@
|
||||
|
||||
objabi.go: s-objabi; @true
|
||||
s-objabi: Makefile
|
||||
rm -f objabi.go.tmp
|
||||
echo "package objabi" > objabi.go.tmp
|
||||
echo "import \"runtime\"" >> objabi.go.tmp
|
||||
echo 'const defaultGOROOT = `$(prefix)`' >> objabi.go.tmp
|
||||
echo 'const defaultGO386 = `sse2`' >> objabi.go.tmp
|
||||
echo 'const defaultGOARM = `5`' >> objabi.go.tmp
|
||||
echo 'const defaultGOMIPS = `hardfloat`' >> objabi.go.tmp
|
||||
echo 'const defaultGOOS = runtime.GOOS' >> objabi.go.tmp
|
||||
echo 'const defaultGOARCH = runtime.GOARCH' >> objabi.go.tmp
|
||||
echo 'const defaultGO_EXTLINK_ENABLED = ``' >> objabi.go.tmp
|
||||
echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> objabi.go.tmp
|
||||
echo 'const stackGuardMultiplier = 1' >> objabi.go.tmp
|
||||
echo 'const goexperiment = ``' >> objabi.go.tmp
|
||||
$(SHELL) $(srcdir)/mvifdiff.sh objabi.go.tmp objabi.go
|
||||
$(STAMP) $@
|
||||
|
||||
runtime_sysinfo.go: s-runtime_sysinfo; @true
|
||||
s-runtime_sysinfo: $(srcdir)/mkrsysinfo.sh gen-sysinfo.go
|
||||
GOARCH=$(GOARCH) GOOS=$(GOOS) $(SHELL) $(srcdir)/mkrsysinfo.sh
|
||||
|
@ -3146,10 +3188,11 @@ zdefaultcc.go: s-zdefaultcc; @true
|
|||
s-zdefaultcc: Makefile
|
||||
echo 'package cfg' > zdefaultcc.go.tmp
|
||||
echo >> zdefaultcc.go.tmp
|
||||
echo 'const DefaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'const DefaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'const DefaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
|
||||
echo 'func DefaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'func DefaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'func DefaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
|
||||
echo 'const DefaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
|
||||
echo 'var OSArchSupportsCgo = map[string]bool{}' >> zdefaultcc.go.tmp
|
||||
$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
|
||||
$(STAMP) $@
|
||||
|
||||
|
@ -3305,6 +3348,7 @@ $(foreach package,$(GOTOOL_PACKAGES),$(eval $(call PACKAGE_template,$(package)))
|
|||
runtime.lo.dep: $(extra_go_files_runtime)
|
||||
syscall.lo.dep: $(extra_go_files_syscall)
|
||||
runtime/internal/sys.lo.dep: $(extra_go_files_runtime_internal_sys)
|
||||
cmd/internal/objabi.lo.dep: $(extra_go_files_cmd_internal_objabi)
|
||||
cmd/go/internal/cfg.lo.dep: $(extra_go_files_cmd_go_internal_cfg)
|
||||
cmd/go/internal/load.lo.dep: $(extra_go_files_cmd_go_internal_load)
|
||||
|
||||
|
|
|
@ -1 +1 @@
|
|||
go1.9
|
||||
go1.10beta1
|
||||
|
|
|
@ -2494,7 +2494,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
|
|||
ac_config_headers="$ac_config_headers config.h"
|
||||
|
||||
|
||||
libtool_VERSION=12:0:0
|
||||
libtool_VERSION=13:0:0
|
||||
|
||||
|
||||
# Default to --enable-multilib
|
||||
|
@ -13652,7 +13652,7 @@ ALLGOARCHFAMILY="I386 ALPHA AMD64 ARM ARM64 IA64 M68K MIPS MIPS64 PPC PPC64 S390
|
|||
|
||||
GOARCH=unknown
|
||||
GOARCH_FAMILY=unknown
|
||||
GOARCH_BIGENDIAN=0
|
||||
GOARCH_BIGENDIAN=false
|
||||
GOARCH_CACHELINESIZE=64
|
||||
GOARCH_PHYSPAGESIZE=4096
|
||||
GOARCH_PCQUANTUM=1
|
||||
|
@ -13680,6 +13680,12 @@ case ${host} in
|
|||
GOARCH_CACHELINESIZE=32
|
||||
GOARCH_PCQUANTUM=4
|
||||
GOARCH_MINFRAMESIZE=4
|
||||
case ${host} in
|
||||
arm*b*-*-*)
|
||||
GOARCH=armbe
|
||||
GOARCH_BIGENDIAN=true
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
i[34567]86-*-* | x86_64-*-*)
|
||||
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
|
||||
|
@ -13712,7 +13718,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
|
|||
m68k*-*-*)
|
||||
GOARCH=m68k
|
||||
GOARCH_FAMILY=M68K
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
GOARCH_CACHELINESIZE=16
|
||||
GOARCH_PCQUANTUM=4
|
||||
GOARCH_INT64ALIGN=2
|
||||
|
@ -13776,7 +13782,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
|
|||
GOARCH="${GOARCH}le"
|
||||
;;
|
||||
*)
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
;;
|
||||
esac
|
||||
GOARCH_CACHELINESIZE=32
|
||||
|
@ -13794,7 +13800,7 @@ _ACEOF
|
|||
if ac_fn_c_try_compile "$LINENO"; then :
|
||||
GOARCH=ppc
|
||||
GOARCH_FAMILY=PPC
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
|
||||
else
|
||||
|
||||
|
@ -13811,7 +13817,7 @@ if ac_fn_c_try_compile "$LINENO"; then :
|
|||
|
||||
else
|
||||
GOARCH=ppc64
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
|
||||
fi
|
||||
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
|
||||
|
@ -13841,7 +13847,7 @@ GOARCH_MINFRAMESIZE=8
|
|||
|
||||
fi
|
||||
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
GOARCH_CACHELINESIZE=256
|
||||
GOARCH_PCQUANTUM=2
|
||||
;;
|
||||
|
@ -13863,7 +13869,7 @@ GOARCH_FAMILY=SPARC64
|
|||
|
||||
fi
|
||||
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
GOARCH_PHYSPAGESIZE=8192
|
||||
GOARCH_PCQUANTUM=4
|
||||
;;
|
||||
|
@ -15142,7 +15148,7 @@ fi
|
|||
$as_echo "$libgo_cv_c_fancymath" >&6; }
|
||||
MATH_FLAG=
|
||||
if test "$libgo_cv_c_fancymath" = yes; then
|
||||
MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations"
|
||||
MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations -fno-math-errno"
|
||||
else
|
||||
MATH_FLAG="-ffp-contract=off"
|
||||
fi
|
||||
|
|
|
@ -11,7 +11,7 @@ AC_INIT(package-unused, version-unused,, libgo)
|
|||
AC_CONFIG_SRCDIR(Makefile.am)
|
||||
AC_CONFIG_HEADER(config.h)
|
||||
|
||||
libtool_VERSION=12:0:0
|
||||
libtool_VERSION=13:0:0
|
||||
AC_SUBST(libtool_VERSION)
|
||||
|
||||
AM_ENABLE_MULTILIB(, ..)
|
||||
|
@ -215,7 +215,7 @@ ALLGOARCHFAMILY="I386 ALPHA AMD64 ARM ARM64 IA64 M68K MIPS MIPS64 PPC PPC64 S390
|
|||
|
||||
GOARCH=unknown
|
||||
GOARCH_FAMILY=unknown
|
||||
GOARCH_BIGENDIAN=0
|
||||
GOARCH_BIGENDIAN=false
|
||||
GOARCH_CACHELINESIZE=64
|
||||
GOARCH_PHYSPAGESIZE=4096
|
||||
GOARCH_PCQUANTUM=1
|
||||
|
@ -243,6 +243,12 @@ case ${host} in
|
|||
GOARCH_CACHELINESIZE=32
|
||||
GOARCH_PCQUANTUM=4
|
||||
GOARCH_MINFRAMESIZE=4
|
||||
case ${host} in
|
||||
arm*b*-*-*)
|
||||
GOARCH=armbe
|
||||
GOARCH_BIGENDIAN=true
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
changequote(,)dnl
|
||||
i[34567]86-*-* | x86_64-*-*)
|
||||
|
@ -270,7 +276,7 @@ GOARCH_HUGEPAGESIZE="1 << 21"
|
|||
m68k*-*-*)
|
||||
GOARCH=m68k
|
||||
GOARCH_FAMILY=M68K
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
GOARCH_CACHELINESIZE=16
|
||||
GOARCH_PCQUANTUM=4
|
||||
GOARCH_INT64ALIGN=2
|
||||
|
@ -313,7 +319,7 @@ GOARCH_HUGEPAGESIZE="1 << 21"
|
|||
GOARCH="${GOARCH}le"
|
||||
;;
|
||||
*)
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
;;
|
||||
esac
|
||||
GOARCH_CACHELINESIZE=32
|
||||
|
@ -327,7 +333,7 @@ GOARCH_HUGEPAGESIZE="1 << 21"
|
|||
#endif],
|
||||
[GOARCH=ppc
|
||||
GOARCH_FAMILY=PPC
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
],
|
||||
[
|
||||
GOARCH_FAMILY=PPC64
|
||||
|
@ -338,7 +344,7 @@ AC_COMPILE_IFELSE([
|
|||
[GOARCH=ppc64le
|
||||
],
|
||||
[GOARCH=ppc64
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
])])
|
||||
GOARCH_PHYSPAGESIZE=65536
|
||||
GOARCH_PCQUANTUM=4
|
||||
|
@ -356,7 +362,7 @@ GOARCH_MINFRAMESIZE=4
|
|||
GOARCH_FAMILY=S390X
|
||||
GOARCH_MINFRAMESIZE=8
|
||||
])
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
GOARCH_CACHELINESIZE=256
|
||||
GOARCH_PCQUANTUM=2
|
||||
;;
|
||||
|
@ -371,7 +377,7 @@ GOARCH_FAMILY=SPARC
|
|||
[GOARCH=sparc64
|
||||
GOARCH_FAMILY=SPARC64
|
||||
])
|
||||
GOARCH_BIGENDIAN=1
|
||||
GOARCH_BIGENDIAN=true
|
||||
GOARCH_PHYSPAGESIZE=8192
|
||||
GOARCH_PCQUANTUM=4
|
||||
;;
|
||||
|
@ -718,7 +724,7 @@ AC_COMPILE_IFELSE([int i;],
|
|||
CFLAGS=$CFLAGS_hold])
|
||||
MATH_FLAG=
|
||||
if test "$libgo_cv_c_fancymath" = yes; then
|
||||
MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations"
|
||||
MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations -fno-math-errno"
|
||||
else
|
||||
MATH_FLAG="-ffp-contract=off"
|
||||
fi
|
||||
|
|
|
@ -3,20 +3,22 @@
|
|||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package tar implements access to tar archives.
|
||||
// It aims to cover most of the variations, including those produced
|
||||
// by GNU and BSD tars.
|
||||
//
|
||||
// References:
|
||||
// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5
|
||||
// http://www.gnu.org/software/tar/manual/html_node/Standard.html
|
||||
// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html
|
||||
// Tape archives (tar) are a file format for storing a sequence of files that
|
||||
// can be read and written in a streaming manner.
|
||||
// This package aims to cover most variations of the format,
|
||||
// including those produced by GNU and BSD tar tools.
|
||||
package tar
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
@ -24,42 +26,500 @@ import (
|
|||
// architectures. If a large value is encountered when decoding, the result
|
||||
// stored in Header will be the truncated version.
|
||||
|
||||
// Header type flags.
|
||||
const (
|
||||
TypeReg = '0' // regular file
|
||||
TypeRegA = '\x00' // regular file
|
||||
TypeLink = '1' // hard link
|
||||
TypeSymlink = '2' // symbolic link
|
||||
TypeChar = '3' // character device node
|
||||
TypeBlock = '4' // block device node
|
||||
TypeDir = '5' // directory
|
||||
TypeFifo = '6' // fifo node
|
||||
TypeCont = '7' // reserved
|
||||
TypeXHeader = 'x' // extended header
|
||||
TypeXGlobalHeader = 'g' // global extended header
|
||||
TypeGNULongName = 'L' // Next file has a long name
|
||||
TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name
|
||||
TypeGNUSparse = 'S' // sparse file
|
||||
var (
|
||||
ErrHeader = errors.New("archive/tar: invalid tar header")
|
||||
ErrWriteTooLong = errors.New("archive/tar: write too long")
|
||||
ErrFieldTooLong = errors.New("archive/tar: header field too long")
|
||||
ErrWriteAfterClose = errors.New("archive/tar: write after close")
|
||||
errMissData = errors.New("archive/tar: sparse file references non-existent data")
|
||||
errUnrefData = errors.New("archive/tar: sparse file contains unreferenced data")
|
||||
errWriteHole = errors.New("archive/tar: write non-NUL byte in sparse hole")
|
||||
)
|
||||
|
||||
type headerError []string
|
||||
|
||||
func (he headerError) Error() string {
|
||||
const prefix = "archive/tar: cannot encode header"
|
||||
var ss []string
|
||||
for _, s := range he {
|
||||
if s != "" {
|
||||
ss = append(ss, s)
|
||||
}
|
||||
}
|
||||
if len(ss) == 0 {
|
||||
return prefix
|
||||
}
|
||||
return fmt.Sprintf("%s: %v", prefix, strings.Join(ss, "; and "))
|
||||
}
|
||||
|
||||
// Type flags for Header.Typeflag.
|
||||
const (
|
||||
// Type '0' indicates a regular file.
|
||||
TypeReg = '0'
|
||||
TypeRegA = '\x00' // For legacy support; use TypeReg instead
|
||||
|
||||
// Type '1' to '6' are header-only flags and may not have a data body.
|
||||
TypeLink = '1' // Hard link
|
||||
TypeSymlink = '2' // Symbolic link
|
||||
TypeChar = '3' // Character device node
|
||||
TypeBlock = '4' // Block device node
|
||||
TypeDir = '5' // Directory
|
||||
TypeFifo = '6' // FIFO node
|
||||
|
||||
// Type '7' is reserved.
|
||||
TypeCont = '7'
|
||||
|
||||
// Type 'x' is used by the PAX format to store key-value records that
|
||||
// are only relevant to the next file.
|
||||
// This package transparently handles these types.
|
||||
TypeXHeader = 'x'
|
||||
|
||||
// Type 'g' is used by the PAX format to store key-value records that
|
||||
// are relevant to all subsequent files.
|
||||
// This package only supports parsing and composing such headers,
|
||||
// but does not currently support persisting the global state across files.
|
||||
TypeXGlobalHeader = 'g'
|
||||
|
||||
// Type 'S' indicates a sparse file in the GNU format.
|
||||
TypeGNUSparse = 'S'
|
||||
|
||||
// Types 'L' and 'K' are used by the GNU format for a meta file
|
||||
// used to store the path or link name for the next file.
|
||||
// This package transparently handles these types.
|
||||
TypeGNULongName = 'L'
|
||||
TypeGNULongLink = 'K'
|
||||
)
|
||||
|
||||
// Keywords for PAX extended header records.
|
||||
const (
|
||||
paxNone = "" // Indicates that no PAX key is suitable
|
||||
paxPath = "path"
|
||||
paxLinkpath = "linkpath"
|
||||
paxSize = "size"
|
||||
paxUid = "uid"
|
||||
paxGid = "gid"
|
||||
paxUname = "uname"
|
||||
paxGname = "gname"
|
||||
paxMtime = "mtime"
|
||||
paxAtime = "atime"
|
||||
paxCtime = "ctime" // Removed from later revision of PAX spec, but was valid
|
||||
paxCharset = "charset" // Currently unused
|
||||
paxComment = "comment" // Currently unused
|
||||
|
||||
paxSchilyXattr = "SCHILY.xattr."
|
||||
|
||||
// Keywords for GNU sparse files in a PAX extended header.
|
||||
paxGNUSparse = "GNU.sparse."
|
||||
paxGNUSparseNumBlocks = "GNU.sparse.numblocks"
|
||||
paxGNUSparseOffset = "GNU.sparse.offset"
|
||||
paxGNUSparseNumBytes = "GNU.sparse.numbytes"
|
||||
paxGNUSparseMap = "GNU.sparse.map"
|
||||
paxGNUSparseName = "GNU.sparse.name"
|
||||
paxGNUSparseMajor = "GNU.sparse.major"
|
||||
paxGNUSparseMinor = "GNU.sparse.minor"
|
||||
paxGNUSparseSize = "GNU.sparse.size"
|
||||
paxGNUSparseRealSize = "GNU.sparse.realsize"
|
||||
)
|
||||
|
||||
// basicKeys is a set of the PAX keys for which we have built-in support.
|
||||
// This does not contain "charset" or "comment", which are both PAX-specific,
|
||||
// so adding them as first-class features of Header is unlikely.
|
||||
// Users can use the PAXRecords field to set it themselves.
|
||||
var basicKeys = map[string]bool{
|
||||
paxPath: true, paxLinkpath: true, paxSize: true, paxUid: true, paxGid: true,
|
||||
paxUname: true, paxGname: true, paxMtime: true, paxAtime: true, paxCtime: true,
|
||||
}
|
||||
|
||||
// A Header represents a single header in a tar archive.
|
||||
// Some fields may not be populated.
|
||||
//
|
||||
// For forward compatibility, users that retrieve a Header from Reader.Next,
|
||||
// mutate it in some ways, and then pass it back to Writer.WriteHeader
|
||||
// should do so by creating a new Header and copying the fields
|
||||
// that they are interested in preserving.
|
||||
type Header struct {
|
||||
Name string // name of header file entry
|
||||
Mode int64 // permission and mode bits
|
||||
Uid int // user id of owner
|
||||
Gid int // group id of owner
|
||||
Size int64 // length in bytes
|
||||
ModTime time.Time // modified time
|
||||
Typeflag byte // type of header entry
|
||||
Linkname string // target name of link
|
||||
Uname string // user name of owner
|
||||
Gname string // group name of owner
|
||||
Devmajor int64 // major number of character or block device
|
||||
Devminor int64 // minor number of character or block device
|
||||
AccessTime time.Time // access time
|
||||
ChangeTime time.Time // status change time
|
||||
Xattrs map[string]string
|
||||
Typeflag byte // Type of header entry (should be TypeReg for most files)
|
||||
|
||||
Name string // Name of file entry
|
||||
Linkname string // Target name of link (valid for TypeLink or TypeSymlink)
|
||||
|
||||
Size int64 // Logical file size in bytes
|
||||
Mode int64 // Permission and mode bits
|
||||
Uid int // User ID of owner
|
||||
Gid int // Group ID of owner
|
||||
Uname string // User name of owner
|
||||
Gname string // Group name of owner
|
||||
|
||||
// If the Format is unspecified, then Writer.WriteHeader rounds ModTime
|
||||
// to the nearest second and ignores the AccessTime and ChangeTime fields.
|
||||
//
|
||||
// To use AccessTime or ChangeTime, specify the Format as PAX or GNU.
|
||||
// To use sub-second resolution, specify the Format as PAX.
|
||||
ModTime time.Time // Modification time
|
||||
AccessTime time.Time // Access time (requires either PAX or GNU support)
|
||||
ChangeTime time.Time // Change time (requires either PAX or GNU support)
|
||||
|
||||
Devmajor int64 // Major device number (valid for TypeChar or TypeBlock)
|
||||
Devminor int64 // Minor device number (valid for TypeChar or TypeBlock)
|
||||
|
||||
// Xattrs stores extended attributes as PAX records under the
|
||||
// "SCHILY.xattr." namespace.
|
||||
//
|
||||
// The following are semantically equivalent:
|
||||
// h.Xattrs[key] = value
|
||||
// h.PAXRecords["SCHILY.xattr."+key] = value
|
||||
//
|
||||
// When Writer.WriteHeader is called, the contents of Xattrs will take
|
||||
// precedence over those in PAXRecords.
|
||||
//
|
||||
// Deprecated: Use PAXRecords instead.
|
||||
Xattrs map[string]string
|
||||
|
||||
// PAXRecords is a map of PAX extended header records.
|
||||
//
|
||||
// User-defined records should have keys of the following form:
|
||||
// VENDOR.keyword
|
||||
// Where VENDOR is some namespace in all uppercase, and keyword may
|
||||
// not contain the '=' character (e.g., "GOLANG.pkg.version").
|
||||
// The key and value should be non-empty UTF-8 strings.
|
||||
//
|
||||
// When Writer.WriteHeader is called, PAX records derived from the
|
||||
// the other fields in Header take precedence over PAXRecords.
|
||||
PAXRecords map[string]string
|
||||
|
||||
// Format specifies the format of the tar header.
|
||||
//
|
||||
// This is set by Reader.Next as a best-effort guess at the format.
|
||||
// Since the Reader liberally reads some non-compliant files,
|
||||
// it is possible for this to be FormatUnknown.
|
||||
//
|
||||
// If the format is unspecified when Writer.WriteHeader is called,
|
||||
// then it uses the first format (in the order of USTAR, PAX, GNU)
|
||||
// capable of encoding this Header (see Format).
|
||||
Format Format
|
||||
}
|
||||
|
||||
// sparseEntry represents a Length-sized fragment at Offset in the file.
|
||||
type sparseEntry struct{ Offset, Length int64 }
|
||||
|
||||
func (s sparseEntry) endOffset() int64 { return s.Offset + s.Length }
|
||||
|
||||
// A sparse file can be represented as either a sparseDatas or a sparseHoles.
|
||||
// As long as the total size is known, they are equivalent and one can be
|
||||
// converted to the other form and back. The various tar formats with sparse
|
||||
// file support represent sparse files in the sparseDatas form. That is, they
|
||||
// specify the fragments in the file that has data, and treat everything else as
|
||||
// having zero bytes. As such, the encoding and decoding logic in this package
|
||||
// deals with sparseDatas.
|
||||
//
|
||||
// However, the external API uses sparseHoles instead of sparseDatas because the
|
||||
// zero value of sparseHoles logically represents a normal file (i.e., there are
|
||||
// no holes in it). On the other hand, the zero value of sparseDatas implies
|
||||
// that the file has no data in it, which is rather odd.
|
||||
//
|
||||
// As an example, if the underlying raw file contains the 10-byte data:
|
||||
// var compactFile = "abcdefgh"
|
||||
//
|
||||
// And the sparse map has the following entries:
|
||||
// var spd sparseDatas = []sparseEntry{
|
||||
// {Offset: 2, Length: 5}, // Data fragment for 2..6
|
||||
// {Offset: 18, Length: 3}, // Data fragment for 18..20
|
||||
// }
|
||||
// var sph sparseHoles = []sparseEntry{
|
||||
// {Offset: 0, Length: 2}, // Hole fragment for 0..1
|
||||
// {Offset: 7, Length: 11}, // Hole fragment for 7..17
|
||||
// {Offset: 21, Length: 4}, // Hole fragment for 21..24
|
||||
// }
|
||||
//
|
||||
// Then the content of the resulting sparse file with a Header.Size of 25 is:
|
||||
// var sparseFile = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4
|
||||
type (
|
||||
sparseDatas []sparseEntry
|
||||
sparseHoles []sparseEntry
|
||||
)
|
||||
|
||||
// validateSparseEntries reports whether sp is a valid sparse map.
|
||||
// It does not matter whether sp represents data fragments or hole fragments.
|
||||
func validateSparseEntries(sp []sparseEntry, size int64) bool {
|
||||
// Validate all sparse entries. These are the same checks as performed by
|
||||
// the BSD tar utility.
|
||||
if size < 0 {
|
||||
return false
|
||||
}
|
||||
var pre sparseEntry
|
||||
for _, cur := range sp {
|
||||
switch {
|
||||
case cur.Offset < 0 || cur.Length < 0:
|
||||
return false // Negative values are never okay
|
||||
case cur.Offset > math.MaxInt64-cur.Length:
|
||||
return false // Integer overflow with large length
|
||||
case cur.endOffset() > size:
|
||||
return false // Region extends beyond the actual size
|
||||
case pre.endOffset() > cur.Offset:
|
||||
return false // Regions cannot overlap and must be in order
|
||||
}
|
||||
pre = cur
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// alignSparseEntries mutates src and returns dst where each fragment's
|
||||
// starting offset is aligned up to the nearest block edge, and each
|
||||
// ending offset is aligned down to the nearest block edge.
|
||||
//
|
||||
// Even though the Go tar Reader and the BSD tar utility can handle entries
|
||||
// with arbitrary offsets and lengths, the GNU tar utility can only handle
|
||||
// offsets and lengths that are multiples of blockSize.
|
||||
func alignSparseEntries(src []sparseEntry, size int64) []sparseEntry {
|
||||
dst := src[:0]
|
||||
for _, s := range src {
|
||||
pos, end := s.Offset, s.endOffset()
|
||||
pos += blockPadding(+pos) // Round-up to nearest blockSize
|
||||
if end != size {
|
||||
end -= blockPadding(-end) // Round-down to nearest blockSize
|
||||
}
|
||||
if pos < end {
|
||||
dst = append(dst, sparseEntry{Offset: pos, Length: end - pos})
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// invertSparseEntries converts a sparse map from one form to the other.
|
||||
// If the input is sparseHoles, then it will output sparseDatas and vice-versa.
|
||||
// The input must have been already validated.
|
||||
//
|
||||
// This function mutates src and returns a normalized map where:
|
||||
// * adjacent fragments are coalesced together
|
||||
// * only the last fragment may be empty
|
||||
// * the endOffset of the last fragment is the total size
|
||||
func invertSparseEntries(src []sparseEntry, size int64) []sparseEntry {
|
||||
dst := src[:0]
|
||||
var pre sparseEntry
|
||||
for _, cur := range src {
|
||||
if cur.Length == 0 {
|
||||
continue // Skip empty fragments
|
||||
}
|
||||
pre.Length = cur.Offset - pre.Offset
|
||||
if pre.Length > 0 {
|
||||
dst = append(dst, pre) // Only add non-empty fragments
|
||||
}
|
||||
pre.Offset = cur.endOffset()
|
||||
}
|
||||
pre.Length = size - pre.Offset // Possibly the only empty fragment
|
||||
return append(dst, pre)
|
||||
}
|
||||
|
||||
// fileState tracks the number of logical (includes sparse holes) and physical
|
||||
// (actual in tar archive) bytes remaining for the current file.
|
||||
//
|
||||
// Invariant: LogicalRemaining >= PhysicalRemaining
|
||||
type fileState interface {
|
||||
LogicalRemaining() int64
|
||||
PhysicalRemaining() int64
|
||||
}
|
||||
|
||||
// allowedFormats determines which formats can be used.
|
||||
// The value returned is the logical OR of multiple possible formats.
|
||||
// If the value is FormatUnknown, then the input Header cannot be encoded
|
||||
// and an error is returned explaining why.
|
||||
//
|
||||
// As a by-product of checking the fields, this function returns paxHdrs, which
|
||||
// contain all fields that could not be directly encoded.
|
||||
// A value receiver ensures that this method does not mutate the source Header.
|
||||
func (h Header) allowedFormats() (format Format, paxHdrs map[string]string, err error) {
|
||||
format = FormatUSTAR | FormatPAX | FormatGNU
|
||||
paxHdrs = make(map[string]string)
|
||||
|
||||
var whyNoUSTAR, whyNoPAX, whyNoGNU string
|
||||
var preferPAX bool // Prefer PAX over USTAR
|
||||
verifyString := func(s string, size int, name, paxKey string) {
|
||||
// NUL-terminator is optional for path and linkpath.
|
||||
// Technically, it is required for uname and gname,
|
||||
// but neither GNU nor BSD tar checks for it.
|
||||
tooLong := len(s) > size
|
||||
allowLongGNU := paxKey == paxPath || paxKey == paxLinkpath
|
||||
if hasNUL(s) || (tooLong && !allowLongGNU) {
|
||||
whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%q", name, s)
|
||||
format.mustNotBe(FormatGNU)
|
||||
}
|
||||
if !isASCII(s) || tooLong {
|
||||
canSplitUSTAR := paxKey == paxPath
|
||||
if _, _, ok := splitUSTARPath(s); !canSplitUSTAR || !ok {
|
||||
whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%q", name, s)
|
||||
format.mustNotBe(FormatUSTAR)
|
||||
}
|
||||
if paxKey == paxNone {
|
||||
whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%q", name, s)
|
||||
format.mustNotBe(FormatPAX)
|
||||
} else {
|
||||
paxHdrs[paxKey] = s
|
||||
}
|
||||
}
|
||||
if v, ok := h.PAXRecords[paxKey]; ok && v == s {
|
||||
paxHdrs[paxKey] = v
|
||||
}
|
||||
}
|
||||
verifyNumeric := func(n int64, size int, name, paxKey string) {
|
||||
if !fitsInBase256(size, n) {
|
||||
whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%d", name, n)
|
||||
format.mustNotBe(FormatGNU)
|
||||
}
|
||||
if !fitsInOctal(size, n) {
|
||||
whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%d", name, n)
|
||||
format.mustNotBe(FormatUSTAR)
|
||||
if paxKey == paxNone {
|
||||
whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%d", name, n)
|
||||
format.mustNotBe(FormatPAX)
|
||||
} else {
|
||||
paxHdrs[paxKey] = strconv.FormatInt(n, 10)
|
||||
}
|
||||
}
|
||||
if v, ok := h.PAXRecords[paxKey]; ok && v == strconv.FormatInt(n, 10) {
|
||||
paxHdrs[paxKey] = v
|
||||
}
|
||||
}
|
||||
verifyTime := func(ts time.Time, size int, name, paxKey string) {
|
||||
if ts.IsZero() {
|
||||
return // Always okay
|
||||
}
|
||||
if !fitsInBase256(size, ts.Unix()) {
|
||||
whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%v", name, ts)
|
||||
format.mustNotBe(FormatGNU)
|
||||
}
|
||||
isMtime := paxKey == paxMtime
|
||||
fitsOctal := fitsInOctal(size, ts.Unix())
|
||||
if (isMtime && !fitsOctal) || !isMtime {
|
||||
whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%v", name, ts)
|
||||
format.mustNotBe(FormatUSTAR)
|
||||
}
|
||||
needsNano := ts.Nanosecond() != 0
|
||||
if !isMtime || !fitsOctal || needsNano {
|
||||
preferPAX = true // USTAR may truncate sub-second measurements
|
||||
if paxKey == paxNone {
|
||||
whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%v", name, ts)
|
||||
format.mustNotBe(FormatPAX)
|
||||
} else {
|
||||
paxHdrs[paxKey] = formatPAXTime(ts)
|
||||
}
|
||||
}
|
||||
if v, ok := h.PAXRecords[paxKey]; ok && v == formatPAXTime(ts) {
|
||||
paxHdrs[paxKey] = v
|
||||
}
|
||||
}
|
||||
|
||||
// Check basic fields.
|
||||
var blk block
|
||||
v7 := blk.V7()
|
||||
ustar := blk.USTAR()
|
||||
gnu := blk.GNU()
|
||||
verifyString(h.Name, len(v7.Name()), "Name", paxPath)
|
||||
verifyString(h.Linkname, len(v7.LinkName()), "Linkname", paxLinkpath)
|
||||
verifyString(h.Uname, len(ustar.UserName()), "Uname", paxUname)
|
||||
verifyString(h.Gname, len(ustar.GroupName()), "Gname", paxGname)
|
||||
verifyNumeric(h.Mode, len(v7.Mode()), "Mode", paxNone)
|
||||
verifyNumeric(int64(h.Uid), len(v7.UID()), "Uid", paxUid)
|
||||
verifyNumeric(int64(h.Gid), len(v7.GID()), "Gid", paxGid)
|
||||
verifyNumeric(h.Size, len(v7.Size()), "Size", paxSize)
|
||||
verifyNumeric(h.Devmajor, len(ustar.DevMajor()), "Devmajor", paxNone)
|
||||
verifyNumeric(h.Devminor, len(ustar.DevMinor()), "Devminor", paxNone)
|
||||
verifyTime(h.ModTime, len(v7.ModTime()), "ModTime", paxMtime)
|
||||
verifyTime(h.AccessTime, len(gnu.AccessTime()), "AccessTime", paxAtime)
|
||||
verifyTime(h.ChangeTime, len(gnu.ChangeTime()), "ChangeTime", paxCtime)
|
||||
|
||||
// Check for header-only types.
|
||||
var whyOnlyPAX, whyOnlyGNU string
|
||||
switch h.Typeflag {
|
||||
case TypeReg, TypeChar, TypeBlock, TypeFifo, TypeGNUSparse:
|
||||
// Exclude TypeLink and TypeSymlink, since they may reference directories.
|
||||
if strings.HasSuffix(h.Name, "/") {
|
||||
return FormatUnknown, nil, headerError{"filename may not have trailing slash"}
|
||||
}
|
||||
case TypeXHeader, TypeGNULongName, TypeGNULongLink:
|
||||
return FormatUnknown, nil, headerError{"cannot manually encode TypeXHeader, TypeGNULongName, or TypeGNULongLink headers"}
|
||||
case TypeXGlobalHeader:
|
||||
h2 := Header{Name: h.Name, Typeflag: h.Typeflag, Xattrs: h.Xattrs, PAXRecords: h.PAXRecords, Format: h.Format}
|
||||
if !reflect.DeepEqual(h, h2) {
|
||||
return FormatUnknown, nil, headerError{"only PAXRecords should be set for TypeXGlobalHeader"}
|
||||
}
|
||||
whyOnlyPAX = "only PAX supports TypeXGlobalHeader"
|
||||
format.mayOnlyBe(FormatPAX)
|
||||
}
|
||||
if !isHeaderOnlyType(h.Typeflag) && h.Size < 0 {
|
||||
return FormatUnknown, nil, headerError{"negative size on header-only type"}
|
||||
}
|
||||
|
||||
// Check PAX records.
|
||||
if len(h.Xattrs) > 0 {
|
||||
for k, v := range h.Xattrs {
|
||||
paxHdrs[paxSchilyXattr+k] = v
|
||||
}
|
||||
whyOnlyPAX = "only PAX supports Xattrs"
|
||||
format.mayOnlyBe(FormatPAX)
|
||||
}
|
||||
if len(h.PAXRecords) > 0 {
|
||||
for k, v := range h.PAXRecords {
|
||||
switch _, exists := paxHdrs[k]; {
|
||||
case exists:
|
||||
continue // Do not overwrite existing records
|
||||
case h.Typeflag == TypeXGlobalHeader:
|
||||
paxHdrs[k] = v // Copy all records
|
||||
case !basicKeys[k] && !strings.HasPrefix(k, paxGNUSparse):
|
||||
paxHdrs[k] = v // Ignore local records that may conflict
|
||||
}
|
||||
}
|
||||
whyOnlyPAX = "only PAX supports PAXRecords"
|
||||
format.mayOnlyBe(FormatPAX)
|
||||
}
|
||||
for k, v := range paxHdrs {
|
||||
if !validPAXRecord(k, v) {
|
||||
return FormatUnknown, nil, headerError{fmt.Sprintf("invalid PAX record: %q", k+" = "+v)}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(dsnet): Re-enable this when adding sparse support.
|
||||
// See https://golang.org/issue/22735
|
||||
/*
|
||||
// Check sparse files.
|
||||
if len(h.SparseHoles) > 0 || h.Typeflag == TypeGNUSparse {
|
||||
if isHeaderOnlyType(h.Typeflag) {
|
||||
return FormatUnknown, nil, headerError{"header-only type cannot be sparse"}
|
||||
}
|
||||
if !validateSparseEntries(h.SparseHoles, h.Size) {
|
||||
return FormatUnknown, nil, headerError{"invalid sparse holes"}
|
||||
}
|
||||
if h.Typeflag == TypeGNUSparse {
|
||||
whyOnlyGNU = "only GNU supports TypeGNUSparse"
|
||||
format.mayOnlyBe(FormatGNU)
|
||||
} else {
|
||||
whyNoGNU = "GNU supports sparse files only with TypeGNUSparse"
|
||||
format.mustNotBe(FormatGNU)
|
||||
}
|
||||
whyNoUSTAR = "USTAR does not support sparse files"
|
||||
format.mustNotBe(FormatUSTAR)
|
||||
}
|
||||
*/
|
||||
|
||||
// Check desired format.
|
||||
if wantFormat := h.Format; wantFormat != FormatUnknown {
|
||||
if wantFormat.has(FormatPAX) && !preferPAX {
|
||||
wantFormat.mayBe(FormatUSTAR) // PAX implies USTAR allowed too
|
||||
}
|
||||
format.mayOnlyBe(wantFormat) // Set union of formats allowed and format wanted
|
||||
}
|
||||
if format == FormatUnknown {
|
||||
switch h.Format {
|
||||
case FormatUSTAR:
|
||||
err = headerError{"Format specifies USTAR", whyNoUSTAR, whyOnlyPAX, whyOnlyGNU}
|
||||
case FormatPAX:
|
||||
err = headerError{"Format specifies PAX", whyNoPAX, whyOnlyGNU}
|
||||
case FormatGNU:
|
||||
err = headerError{"Format specifies GNU", whyNoGNU, whyOnlyPAX}
|
||||
default:
|
||||
err = headerError{whyNoUSTAR, whyNoPAX, whyNoGNU, whyOnlyPAX, whyOnlyGNU}
|
||||
}
|
||||
}
|
||||
return format, paxHdrs, err
|
||||
}
|
||||
|
||||
// FileInfo returns an os.FileInfo for the Header.
|
||||
|
@ -92,63 +552,43 @@ func (fi headerFileInfo) Mode() (mode os.FileMode) {
|
|||
|
||||
// Set setuid, setgid and sticky bits.
|
||||
if fi.h.Mode&c_ISUID != 0 {
|
||||
// setuid
|
||||
mode |= os.ModeSetuid
|
||||
}
|
||||
if fi.h.Mode&c_ISGID != 0 {
|
||||
// setgid
|
||||
mode |= os.ModeSetgid
|
||||
}
|
||||
if fi.h.Mode&c_ISVTX != 0 {
|
||||
// sticky
|
||||
mode |= os.ModeSticky
|
||||
}
|
||||
|
||||
// Set file mode bits.
|
||||
// clear perm, setuid, setgid and sticky bits.
|
||||
m := os.FileMode(fi.h.Mode) &^ 07777
|
||||
if m == c_ISDIR {
|
||||
// directory
|
||||
// Set file mode bits; clear perm, setuid, setgid, and sticky bits.
|
||||
switch m := os.FileMode(fi.h.Mode) &^ 07777; m {
|
||||
case c_ISDIR:
|
||||
mode |= os.ModeDir
|
||||
}
|
||||
if m == c_ISFIFO {
|
||||
// named pipe (FIFO)
|
||||
case c_ISFIFO:
|
||||
mode |= os.ModeNamedPipe
|
||||
}
|
||||
if m == c_ISLNK {
|
||||
// symbolic link
|
||||
case c_ISLNK:
|
||||
mode |= os.ModeSymlink
|
||||
}
|
||||
if m == c_ISBLK {
|
||||
// device file
|
||||
case c_ISBLK:
|
||||
mode |= os.ModeDevice
|
||||
}
|
||||
if m == c_ISCHR {
|
||||
// Unix character device
|
||||
case c_ISCHR:
|
||||
mode |= os.ModeDevice
|
||||
mode |= os.ModeCharDevice
|
||||
}
|
||||
if m == c_ISSOCK {
|
||||
// Unix domain socket
|
||||
case c_ISSOCK:
|
||||
mode |= os.ModeSocket
|
||||
}
|
||||
|
||||
switch fi.h.Typeflag {
|
||||
case TypeSymlink:
|
||||
// symbolic link
|
||||
mode |= os.ModeSymlink
|
||||
case TypeChar:
|
||||
// character device node
|
||||
mode |= os.ModeDevice
|
||||
mode |= os.ModeCharDevice
|
||||
case TypeBlock:
|
||||
// block device node
|
||||
mode |= os.ModeDevice
|
||||
case TypeDir:
|
||||
// directory
|
||||
mode |= os.ModeDir
|
||||
case TypeFifo:
|
||||
// fifo node
|
||||
mode |= os.ModeNamedPipe
|
||||
}
|
||||
|
||||
|
@ -176,33 +616,16 @@ const (
|
|||
c_ISSOCK = 0140000 // Socket
|
||||
)
|
||||
|
||||
// Keywords for the PAX Extended Header
|
||||
const (
|
||||
paxAtime = "atime"
|
||||
paxCharset = "charset"
|
||||
paxComment = "comment"
|
||||
paxCtime = "ctime" // please note that ctime is not a valid pax header.
|
||||
paxGid = "gid"
|
||||
paxGname = "gname"
|
||||
paxLinkpath = "linkpath"
|
||||
paxMtime = "mtime"
|
||||
paxPath = "path"
|
||||
paxSize = "size"
|
||||
paxUid = "uid"
|
||||
paxUname = "uname"
|
||||
paxXattr = "SCHILY.xattr."
|
||||
paxNone = ""
|
||||
)
|
||||
|
||||
// FileInfoHeader creates a partially-populated Header from fi.
|
||||
// If fi describes a symlink, FileInfoHeader records link as the link target.
|
||||
// If fi describes a directory, a slash is appended to the name.
|
||||
// Because os.FileInfo's Name method returns only the base name of
|
||||
// the file it describes, it may be necessary to modify the Name field
|
||||
// of the returned header to provide the full path name of the file.
|
||||
//
|
||||
// Since os.FileInfo's Name method only returns the base name of
|
||||
// the file it describes, it may be necessary to modify Header.Name
|
||||
// to provide the full path name of the file.
|
||||
func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
|
||||
if fi == nil {
|
||||
return nil, errors.New("tar: FileInfo is nil")
|
||||
return nil, errors.New("archive/tar: FileInfo is nil")
|
||||
}
|
||||
fm := fi.Mode()
|
||||
h := &Header{
|
||||
|
@ -265,6 +688,12 @@ func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
|
|||
h.Size = 0
|
||||
h.Linkname = sys.Linkname
|
||||
}
|
||||
if sys.PAXRecords != nil {
|
||||
h.PAXRecords = make(map[string]string)
|
||||
for k, v := range sys.PAXRecords {
|
||||
h.PAXRecords[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if sysStat != nil {
|
||||
return h, sysStat(fi, h)
|
||||
|
@ -282,3 +711,10 @@ func isHeaderOnlyType(flag byte) bool {
|
|||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func min(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
|
|
@ -4,38 +4,133 @@
|
|||
|
||||
package tar
|
||||
|
||||
import "strings"
|
||||
|
||||
// Format represents the tar archive format.
|
||||
//
|
||||
// The original tar format was introduced in Unix V7.
|
||||
// Since then, there have been multiple competing formats attempting to
|
||||
// standardize or extend the V7 format to overcome its limitations.
|
||||
// The most common formats are the USTAR, PAX, and GNU formats,
|
||||
// each with their own advantages and limitations.
|
||||
//
|
||||
// The following table captures the capabilities of each format:
|
||||
//
|
||||
// | USTAR | PAX | GNU
|
||||
// ------------------+--------+-----------+----------
|
||||
// Name | 256B | unlimited | unlimited
|
||||
// Linkname | 100B | unlimited | unlimited
|
||||
// Size | uint33 | unlimited | uint89
|
||||
// Mode | uint21 | uint21 | uint57
|
||||
// Uid/Gid | uint21 | unlimited | uint57
|
||||
// Uname/Gname | 32B | unlimited | 32B
|
||||
// ModTime | uint33 | unlimited | int89
|
||||
// AccessTime | n/a | unlimited | int89
|
||||
// ChangeTime | n/a | unlimited | int89
|
||||
// Devmajor/Devminor | uint21 | uint21 | uint57
|
||||
// ------------------+--------+-----------+----------
|
||||
// string encoding | ASCII | UTF-8 | binary
|
||||
// sub-second times | no | yes | no
|
||||
// sparse files | no | yes | yes
|
||||
//
|
||||
// The table's upper portion shows the Header fields, where each format reports
|
||||
// the maximum number of bytes allowed for each string field and
|
||||
// the integer type used to store each numeric field
|
||||
// (where timestamps are stored as the number of seconds since the Unix epoch).
|
||||
//
|
||||
// The table's lower portion shows specialized features of each format,
|
||||
// such as supported string encodings, support for sub-second timestamps,
|
||||
// or support for sparse files.
|
||||
//
|
||||
// The Writer currently provides no support for sparse files.
|
||||
type Format int
|
||||
|
||||
// Constants to identify various tar formats.
|
||||
const (
|
||||
// The format is unknown.
|
||||
formatUnknown = (1 << iota) / 2 // Sequence of 0, 1, 2, 4, 8, etc...
|
||||
// Deliberately hide the meaning of constants from public API.
|
||||
_ Format = (1 << iota) / 4 // Sequence of 0, 0, 1, 2, 4, 8, etc...
|
||||
|
||||
// FormatUnknown indicates that the format is unknown.
|
||||
FormatUnknown
|
||||
|
||||
// The format of the original Unix V7 tar tool prior to standardization.
|
||||
formatV7
|
||||
|
||||
// The old and new GNU formats, which are incompatible with USTAR.
|
||||
// This does cover the old GNU sparse extension.
|
||||
// This does not cover the GNU sparse extensions using PAX headers,
|
||||
// versions 0.0, 0.1, and 1.0; these fall under the PAX format.
|
||||
formatGNU
|
||||
// FormatUSTAR represents the USTAR header format defined in POSIX.1-1988.
|
||||
//
|
||||
// While this format is compatible with most tar readers,
|
||||
// the format has several limitations making it unsuitable for some usages.
|
||||
// Most notably, it cannot support sparse files, files larger than 8GiB,
|
||||
// filenames larger than 256 characters, and non-ASCII filenames.
|
||||
//
|
||||
// Reference:
|
||||
// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06
|
||||
FormatUSTAR
|
||||
|
||||
// FormatPAX represents the PAX header format defined in POSIX.1-2001.
|
||||
//
|
||||
// PAX extends USTAR by writing a special file with Typeflag TypeXHeader
|
||||
// preceding the original header. This file contains a set of key-value
|
||||
// records, which are used to overcome USTAR's shortcomings, in addition to
|
||||
// providing the ability to have sub-second resolution for timestamps.
|
||||
//
|
||||
// Some newer formats add their own extensions to PAX by defining their
|
||||
// own keys and assigning certain semantic meaning to the associated values.
|
||||
// For example, sparse file support in PAX is implemented using keys
|
||||
// defined by the GNU manual (e.g., "GNU.sparse.map").
|
||||
//
|
||||
// Reference:
|
||||
// http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html
|
||||
FormatPAX
|
||||
|
||||
// FormatGNU represents the GNU header format.
|
||||
//
|
||||
// The GNU header format is older than the USTAR and PAX standards and
|
||||
// is not compatible with them. The GNU format supports
|
||||
// arbitrary file sizes, filenames of arbitrary encoding and length,
|
||||
// sparse files, and other features.
|
||||
//
|
||||
// It is recommended that PAX be chosen over GNU unless the target
|
||||
// application can only parse GNU formatted archives.
|
||||
//
|
||||
// Reference:
|
||||
// http://www.gnu.org/software/tar/manual/html_node/Standard.html
|
||||
FormatGNU
|
||||
|
||||
// Schily's tar format, which is incompatible with USTAR.
|
||||
// This does not cover STAR extensions to the PAX format; these fall under
|
||||
// the PAX format.
|
||||
formatSTAR
|
||||
|
||||
// USTAR is the former standardization of tar defined in POSIX.1-1988.
|
||||
// This is incompatible with the GNU and STAR formats.
|
||||
formatUSTAR
|
||||
|
||||
// PAX is the latest standardization of tar defined in POSIX.1-2001.
|
||||
// This is an extension of USTAR and is "backwards compatible" with it.
|
||||
//
|
||||
// Some newer formats add their own extensions to PAX, such as GNU sparse
|
||||
// files and SCHILY extended attributes. Since they are backwards compatible
|
||||
// with PAX, they will be labelled as "PAX".
|
||||
formatPAX
|
||||
formatMax
|
||||
)
|
||||
|
||||
func (f Format) has(f2 Format) bool { return f&f2 != 0 }
|
||||
func (f *Format) mayBe(f2 Format) { *f |= f2 }
|
||||
func (f *Format) mayOnlyBe(f2 Format) { *f &= f2 }
|
||||
func (f *Format) mustNotBe(f2 Format) { *f &^= f2 }
|
||||
|
||||
var formatNames = map[Format]string{
|
||||
formatV7: "V7", FormatUSTAR: "USTAR", FormatPAX: "PAX", FormatGNU: "GNU", formatSTAR: "STAR",
|
||||
}
|
||||
|
||||
func (f Format) String() string {
|
||||
var ss []string
|
||||
for f2 := Format(1); f2 < formatMax; f2 <<= 1 {
|
||||
if f.has(f2) {
|
||||
ss = append(ss, formatNames[f2])
|
||||
}
|
||||
}
|
||||
switch len(ss) {
|
||||
case 0:
|
||||
return "<unknown>"
|
||||
case 1:
|
||||
return ss[0]
|
||||
default:
|
||||
return "(" + strings.Join(ss, " | ") + ")"
|
||||
}
|
||||
}
|
||||
|
||||
// Magics used to identify various formats.
|
||||
const (
|
||||
magicGNU, versionGNU = "ustar ", " \x00"
|
||||
|
@ -50,6 +145,12 @@ const (
|
|||
prefixSize = 155 // Max length of the prefix field in USTAR format
|
||||
)
|
||||
|
||||
// blockPadding computes the number of bytes needed to pad offset up to the
|
||||
// nearest block edge where 0 <= n < blockSize.
|
||||
func blockPadding(offset int64) (n int64) {
|
||||
return -offset & (blockSize - 1)
|
||||
}
|
||||
|
||||
var zeroBlock block
|
||||
|
||||
type block [blockSize]byte
|
||||
|
@ -63,14 +164,14 @@ func (b *block) Sparse() sparseArray { return (sparseArray)(b[:]) }
|
|||
|
||||
// GetFormat checks that the block is a valid tar header based on the checksum.
|
||||
// It then attempts to guess the specific format based on magic values.
|
||||
// If the checksum fails, then formatUnknown is returned.
|
||||
func (b *block) GetFormat() (format int) {
|
||||
// If the checksum fails, then FormatUnknown is returned.
|
||||
func (b *block) GetFormat() Format {
|
||||
// Verify checksum.
|
||||
var p parser
|
||||
value := p.parseOctal(b.V7().Chksum())
|
||||
chksum1, chksum2 := b.ComputeChecksum()
|
||||
if p.err != nil || (value != chksum1 && value != chksum2) {
|
||||
return formatUnknown
|
||||
return FormatUnknown
|
||||
}
|
||||
|
||||
// Guess the magic values.
|
||||
|
@ -81,9 +182,9 @@ func (b *block) GetFormat() (format int) {
|
|||
case magic == magicUSTAR && trailer == trailerSTAR:
|
||||
return formatSTAR
|
||||
case magic == magicUSTAR:
|
||||
return formatUSTAR
|
||||
return FormatUSTAR | FormatPAX
|
||||
case magic == magicGNU && version == versionGNU:
|
||||
return formatGNU
|
||||
return FormatGNU
|
||||
default:
|
||||
return formatV7
|
||||
}
|
||||
|
@ -91,19 +192,19 @@ func (b *block) GetFormat() (format int) {
|
|||
|
||||
// SetFormat writes the magic values necessary for specified format
|
||||
// and then updates the checksum accordingly.
|
||||
func (b *block) SetFormat(format int) {
|
||||
func (b *block) SetFormat(format Format) {
|
||||
// Set the magic values.
|
||||
switch format {
|
||||
case formatV7:
|
||||
switch {
|
||||
case format.has(formatV7):
|
||||
// Do nothing.
|
||||
case formatGNU:
|
||||
case format.has(FormatGNU):
|
||||
copy(b.GNU().Magic(), magicGNU)
|
||||
copy(b.GNU().Version(), versionGNU)
|
||||
case formatSTAR:
|
||||
case format.has(formatSTAR):
|
||||
copy(b.STAR().Magic(), magicUSTAR)
|
||||
copy(b.STAR().Version(), versionUSTAR)
|
||||
copy(b.STAR().Trailer(), trailerSTAR)
|
||||
case formatUSTAR, formatPAX:
|
||||
case format.has(FormatUSTAR | FormatPAX):
|
||||
copy(b.USTAR().Magic(), magicUSTAR)
|
||||
copy(b.USTAR().Version(), versionUSTAR)
|
||||
default:
|
||||
|
@ -128,12 +229,17 @@ func (b *block) ComputeChecksum() (unsigned, signed int64) {
|
|||
if 148 <= i && i < 156 {
|
||||
c = ' ' // Treat the checksum field itself as all spaces.
|
||||
}
|
||||
unsigned += int64(uint8(c))
|
||||
unsigned += int64(c)
|
||||
signed += int64(int8(c))
|
||||
}
|
||||
return unsigned, signed
|
||||
}
|
||||
|
||||
// Reset clears the block with all zeros.
|
||||
func (b *block) Reset() {
|
||||
*b = block{}
|
||||
}
|
||||
|
||||
type headerV7 [blockSize]byte
|
||||
|
||||
func (h *headerV7) Name() []byte { return h[000:][:100] }
|
||||
|
@ -187,11 +293,11 @@ func (h *headerUSTAR) Prefix() []byte { return h[345:][:155] }
|
|||
|
||||
type sparseArray []byte
|
||||
|
||||
func (s sparseArray) Entry(i int) sparseNode { return (sparseNode)(s[i*24:]) }
|
||||
func (s sparseArray) Entry(i int) sparseElem { return (sparseElem)(s[i*24:]) }
|
||||
func (s sparseArray) IsExtended() []byte { return s[24*s.MaxEntries():][:1] }
|
||||
func (s sparseArray) MaxEntries() int { return len(s) / 24 }
|
||||
|
||||
type sparseNode []byte
|
||||
type sparseElem []byte
|
||||
|
||||
func (s sparseNode) Offset() []byte { return s[00:][:12] }
|
||||
func (s sparseNode) NumBytes() []byte { return s[12:][:12] }
|
||||
func (s sparseElem) Offset() []byte { return s[00:][:12] }
|
||||
func (s sparseElem) Length() []byte { return s[12:][:12] }
|
||||
|
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -8,6 +8,10 @@ package tar
|
|||
|
||||
import (
|
||||
"os"
|
||||
"os/user"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
|
@ -15,6 +19,10 @@ func init() {
|
|||
sysStat = statUnix
|
||||
}
|
||||
|
||||
// userMap and groupMap caches UID and GID lookups for performance reasons.
|
||||
// The downside is that renaming uname or gname by the OS never takes effect.
|
||||
var userMap, groupMap sync.Map // map[int]string
|
||||
|
||||
func statUnix(fi os.FileInfo, h *Header) error {
|
||||
sys, ok := fi.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
|
@ -22,11 +30,67 @@ func statUnix(fi os.FileInfo, h *Header) error {
|
|||
}
|
||||
h.Uid = int(sys.Uid)
|
||||
h.Gid = int(sys.Gid)
|
||||
// TODO(bradfitz): populate username & group. os/user
|
||||
// doesn't cache LookupId lookups, and lacks group
|
||||
// lookup functions.
|
||||
|
||||
// Best effort at populating Uname and Gname.
|
||||
// The os/user functions may fail for any number of reasons
|
||||
// (not implemented on that platform, cgo not enabled, etc).
|
||||
if u, ok := userMap.Load(h.Uid); ok {
|
||||
h.Uname = u.(string)
|
||||
} else if u, err := user.LookupId(strconv.Itoa(h.Uid)); err == nil {
|
||||
h.Uname = u.Username
|
||||
userMap.Store(h.Uid, h.Uname)
|
||||
}
|
||||
if g, ok := groupMap.Load(h.Gid); ok {
|
||||
h.Gname = g.(string)
|
||||
} else if g, err := user.LookupGroupId(strconv.Itoa(h.Gid)); err == nil {
|
||||
h.Gname = g.Name
|
||||
groupMap.Store(h.Gid, h.Gname)
|
||||
}
|
||||
|
||||
h.AccessTime = statAtime(sys)
|
||||
h.ChangeTime = statCtime(sys)
|
||||
// TODO(bradfitz): major/minor device numbers?
|
||||
|
||||
// Best effort at populating Devmajor and Devminor.
|
||||
if h.Typeflag == TypeChar || h.Typeflag == TypeBlock {
|
||||
dev := uint64(sys.Rdev) // May be int32 or uint32
|
||||
switch runtime.GOOS {
|
||||
case "linux":
|
||||
// Copied from golang.org/x/sys/unix/dev_linux.go.
|
||||
major := uint32((dev & 0x00000000000fff00) >> 8)
|
||||
major |= uint32((dev & 0xfffff00000000000) >> 32)
|
||||
minor := uint32((dev & 0x00000000000000ff) >> 0)
|
||||
minor |= uint32((dev & 0x00000ffffff00000) >> 12)
|
||||
h.Devmajor, h.Devminor = int64(major), int64(minor)
|
||||
case "darwin":
|
||||
// Copied from golang.org/x/sys/unix/dev_darwin.go.
|
||||
major := uint32((dev >> 24) & 0xff)
|
||||
minor := uint32(dev & 0xffffff)
|
||||
h.Devmajor, h.Devminor = int64(major), int64(minor)
|
||||
case "dragonfly":
|
||||
// Copied from golang.org/x/sys/unix/dev_dragonfly.go.
|
||||
major := uint32((dev >> 8) & 0xff)
|
||||
minor := uint32(dev & 0xffff00ff)
|
||||
h.Devmajor, h.Devminor = int64(major), int64(minor)
|
||||
case "freebsd":
|
||||
// Copied from golang.org/x/sys/unix/dev_freebsd.go.
|
||||
major := uint32((dev >> 8) & 0xff)
|
||||
minor := uint32(dev & 0xffff00ff)
|
||||
h.Devmajor, h.Devminor = int64(major), int64(minor)
|
||||
case "netbsd":
|
||||
// Copied from golang.org/x/sys/unix/dev_netbsd.go.
|
||||
major := uint32((dev & 0x000fff00) >> 8)
|
||||
minor := uint32((dev & 0x000000ff) >> 0)
|
||||
minor |= uint32((dev & 0xfff00000) >> 12)
|
||||
h.Devmajor, h.Devminor = int64(major), int64(minor)
|
||||
case "openbsd":
|
||||
// Copied from golang.org/x/sys/unix/dev_openbsd.go.
|
||||
major := uint32((dev & 0x0000ff00) >> 8)
|
||||
minor := uint32((dev & 0x000000ff) >> 0)
|
||||
minor |= uint32((dev & 0xffff0000) >> 8)
|
||||
h.Devmajor, h.Devminor = int64(major), int64(minor)
|
||||
default:
|
||||
// TODO: Implement solaris (see https://golang.org/issue/8106)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -12,26 +12,34 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
// hasNUL reports whether the NUL character exists within s.
|
||||
func hasNUL(s string) bool {
|
||||
return strings.IndexByte(s, 0) >= 0
|
||||
}
|
||||
|
||||
// isASCII reports whether the input is an ASCII C-style string.
|
||||
func isASCII(s string) bool {
|
||||
for _, c := range s {
|
||||
if c >= 0x80 {
|
||||
if c >= 0x80 || c == 0x00 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// toASCII converts the input to an ASCII C-style string.
|
||||
// This a best effort conversion, so invalid characters are dropped.
|
||||
func toASCII(s string) string {
|
||||
if isASCII(s) {
|
||||
return s
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
b := make([]byte, 0, len(s))
|
||||
for _, c := range s {
|
||||
if c < 0x80 {
|
||||
buf.WriteByte(byte(c))
|
||||
if c < 0x80 && c != 0x00 {
|
||||
b = append(b, byte(c))
|
||||
}
|
||||
}
|
||||
return buf.String()
|
||||
return string(b)
|
||||
}
|
||||
|
||||
type parser struct {
|
||||
|
@ -45,23 +53,28 @@ type formatter struct {
|
|||
// parseString parses bytes as a NUL-terminated C-style string.
|
||||
// If a NUL byte is not found then the whole slice is returned as a string.
|
||||
func (*parser) parseString(b []byte) string {
|
||||
n := 0
|
||||
for n < len(b) && b[n] != 0 {
|
||||
n++
|
||||
if i := bytes.IndexByte(b, 0); i >= 0 {
|
||||
return string(b[:i])
|
||||
}
|
||||
return string(b[0:n])
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// Write s into b, terminating it with a NUL if there is room.
|
||||
// formatString copies s into b, NUL-terminating if possible.
|
||||
func (f *formatter) formatString(b []byte, s string) {
|
||||
if len(s) > len(b) {
|
||||
f.err = ErrFieldTooLong
|
||||
return
|
||||
}
|
||||
ascii := toASCII(s)
|
||||
copy(b, ascii)
|
||||
if len(ascii) < len(b) {
|
||||
b[len(ascii)] = 0
|
||||
copy(b, s)
|
||||
if len(s) < len(b) {
|
||||
b[len(s)] = 0
|
||||
}
|
||||
|
||||
// Some buggy readers treat regular files with a trailing slash
|
||||
// in the V7 path field as a directory even though the full path
|
||||
// recorded elsewhere (e.g., via PAX record) contains no trailing slash.
|
||||
if len(s) > len(b) && b[len(b)-1] == '/' {
|
||||
n := len(strings.TrimRight(s[:len(b)], "/"))
|
||||
b[n] = 0 // Replace trailing slash with NUL terminator
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -73,7 +86,7 @@ func (f *formatter) formatString(b []byte, s string) {
|
|||
// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
|
||||
// equivalent to the sign bit in two's complement form.
|
||||
func fitsInBase256(n int, x int64) bool {
|
||||
var binBits = uint(n-1) * 8
|
||||
binBits := uint(n-1) * 8
|
||||
return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
|
||||
}
|
||||
|
||||
|
@ -121,8 +134,14 @@ func (p *parser) parseNumeric(b []byte) int64 {
|
|||
return p.parseOctal(b)
|
||||
}
|
||||
|
||||
// Write x into b, as binary (GNUtar/star extension).
|
||||
// formatNumeric encodes x into b using base-8 (octal) encoding if possible.
|
||||
// Otherwise it will attempt to use base-256 (binary) encoding.
|
||||
func (f *formatter) formatNumeric(b []byte, x int64) {
|
||||
if fitsInOctal(len(b), x) {
|
||||
f.formatOctal(b, x)
|
||||
return
|
||||
}
|
||||
|
||||
if fitsInBase256(len(b), x) {
|
||||
for i := len(b) - 1; i >= 0; i-- {
|
||||
b[i] = byte(x)
|
||||
|
@ -155,6 +174,11 @@ func (p *parser) parseOctal(b []byte) int64 {
|
|||
}
|
||||
|
||||
func (f *formatter) formatOctal(b []byte, x int64) {
|
||||
if !fitsInOctal(len(b), x) {
|
||||
x = 0 // Last resort, just write zero
|
||||
f.err = ErrFieldTooLong
|
||||
}
|
||||
|
||||
s := strconv.FormatInt(x, 8)
|
||||
// Add leading zeros, but leave room for a NUL.
|
||||
if n := len(b) - len(s) - 1; n > 0 {
|
||||
|
@ -163,6 +187,13 @@ func (f *formatter) formatOctal(b []byte, x int64) {
|
|||
f.formatString(b, s)
|
||||
}
|
||||
|
||||
// fitsInOctal reports whether the integer x fits in a field n-bytes long
|
||||
// using octal encoding with the appropriate NUL terminator.
|
||||
func fitsInOctal(n int, x int64) bool {
|
||||
octBits := uint(n-1) * 3
|
||||
return x >= 0 && (n >= 22 || x < 1<<octBits)
|
||||
}
|
||||
|
||||
// parsePAXTime takes a string of the form %d.%d as described in the PAX
|
||||
// specification. Note that this implementation allows for negative timestamps,
|
||||
// which is allowed for by the PAX specification, but not always portable.
|
||||
|
@ -195,19 +226,32 @@ func parsePAXTime(s string) (time.Time, error) {
|
|||
}
|
||||
nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed
|
||||
if len(ss) > 0 && ss[0] == '-' {
|
||||
return time.Unix(secs, -1*int64(nsecs)), nil // Negative correction
|
||||
return time.Unix(secs, -1*nsecs), nil // Negative correction
|
||||
}
|
||||
return time.Unix(secs, int64(nsecs)), nil
|
||||
return time.Unix(secs, nsecs), nil
|
||||
}
|
||||
|
||||
// TODO(dsnet): Implement formatPAXTime.
|
||||
// formatPAXTime converts ts into a time of the form %d.%d as described in the
|
||||
// PAX specification. This function is capable of negative timestamps.
|
||||
func formatPAXTime(ts time.Time) (s string) {
|
||||
secs, nsecs := ts.Unix(), ts.Nanosecond()
|
||||
if nsecs == 0 {
|
||||
return strconv.FormatInt(secs, 10)
|
||||
}
|
||||
|
||||
// If seconds is negative, then perform correction.
|
||||
sign := ""
|
||||
if secs < 0 {
|
||||
sign = "-" // Remember sign
|
||||
secs = -(secs + 1) // Add a second to secs
|
||||
nsecs = -(nsecs - 1E9) // Take that second away from nsecs
|
||||
}
|
||||
return strings.TrimRight(fmt.Sprintf("%s%d.%09d", sign, secs, nsecs), "0")
|
||||
}
|
||||
|
||||
// parsePAXRecord parses the input PAX record string into a key-value pair.
|
||||
// If parsing is successful, it will slice off the currently read record and
|
||||
// return the remainder as r.
|
||||
//
|
||||
// A PAX record is of the following form:
|
||||
// "%d %s=%s\n" % (size, key, value)
|
||||
func parsePAXRecord(s string) (k, v, r string, err error) {
|
||||
// The size field ends at the first space.
|
||||
sp := strings.IndexByte(s, ' ')
|
||||
|
@ -232,21 +276,51 @@ func parsePAXRecord(s string) (k, v, r string, err error) {
|
|||
if eq == -1 {
|
||||
return "", "", s, ErrHeader
|
||||
}
|
||||
return rec[:eq], rec[eq+1:], rem, nil
|
||||
k, v = rec[:eq], rec[eq+1:]
|
||||
|
||||
if !validPAXRecord(k, v) {
|
||||
return "", "", s, ErrHeader
|
||||
}
|
||||
return k, v, rem, nil
|
||||
}
|
||||
|
||||
// formatPAXRecord formats a single PAX record, prefixing it with the
|
||||
// appropriate length.
|
||||
func formatPAXRecord(k, v string) string {
|
||||
func formatPAXRecord(k, v string) (string, error) {
|
||||
if !validPAXRecord(k, v) {
|
||||
return "", ErrHeader
|
||||
}
|
||||
|
||||
const padding = 3 // Extra padding for ' ', '=', and '\n'
|
||||
size := len(k) + len(v) + padding
|
||||
size += len(strconv.Itoa(size))
|
||||
record := fmt.Sprintf("%d %s=%s\n", size, k, v)
|
||||
record := strconv.Itoa(size) + " " + k + "=" + v + "\n"
|
||||
|
||||
// Final adjustment if adding size field increased the record size.
|
||||
if len(record) != size {
|
||||
size = len(record)
|
||||
record = fmt.Sprintf("%d %s=%s\n", size, k, v)
|
||||
record = strconv.Itoa(size) + " " + k + "=" + v + "\n"
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// validPAXRecord reports whether the key-value pair is valid where each
|
||||
// record is formatted as:
|
||||
// "%d %s=%s\n" % (size, key, value)
|
||||
//
|
||||
// Keys and values should be UTF-8, but the number of bad writers out there
|
||||
// forces us to be a more liberal.
|
||||
// Thus, we only reject all keys with NUL, and only reject NULs in values
|
||||
// for the PAX version of the USTAR string fields.
|
||||
// The key must not contain an '=' character.
|
||||
func validPAXRecord(k, v string) bool {
|
||||
if k == "" || strings.IndexByte(k, '=') >= 0 {
|
||||
return false
|
||||
}
|
||||
switch k {
|
||||
case paxPath, paxLinkpath, paxUname, paxGname:
|
||||
return !hasNUL(v)
|
||||
default:
|
||||
return !hasNUL(k)
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
|
|
@ -110,6 +110,25 @@ func TestFormatNumeric(t *testing.T) {
|
|||
want string
|
||||
ok bool
|
||||
}{
|
||||
// Test base-8 (octal) encoded values.
|
||||
{0, "0\x00", true},
|
||||
{7, "7\x00", true},
|
||||
{8, "\x80\x08", true},
|
||||
{077, "77\x00", true},
|
||||
{0100, "\x80\x00\x40", true},
|
||||
{0, "0000000\x00", true},
|
||||
{0123, "0000123\x00", true},
|
||||
{07654321, "7654321\x00", true},
|
||||
{07777777, "7777777\x00", true},
|
||||
{010000000, "\x80\x00\x00\x00\x00\x20\x00\x00", true},
|
||||
{0, "00000000000\x00", true},
|
||||
{000001234567, "00001234567\x00", true},
|
||||
{076543210321, "76543210321\x00", true},
|
||||
{012345670123, "12345670123\x00", true},
|
||||
{077777777777, "77777777777\x00", true},
|
||||
{0100000000000, "\x80\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00", true},
|
||||
{math.MaxInt64, "777777777777777777777\x00", true},
|
||||
|
||||
// Test base-256 (binary) encoded values.
|
||||
{-1, "\xff", true},
|
||||
{-1, "\xff\xff", true},
|
||||
|
@ -155,6 +174,45 @@ func TestFormatNumeric(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestFitsInOctal(t *testing.T) {
|
||||
vectors := []struct {
|
||||
input int64
|
||||
width int
|
||||
ok bool
|
||||
}{
|
||||
{-1, 1, false},
|
||||
{-1, 2, false},
|
||||
{-1, 3, false},
|
||||
{0, 1, true},
|
||||
{0 + 1, 1, false},
|
||||
{0, 2, true},
|
||||
{07, 2, true},
|
||||
{07 + 1, 2, false},
|
||||
{0, 4, true},
|
||||
{0777, 4, true},
|
||||
{0777 + 1, 4, false},
|
||||
{0, 8, true},
|
||||
{07777777, 8, true},
|
||||
{07777777 + 1, 8, false},
|
||||
{0, 12, true},
|
||||
{077777777777, 12, true},
|
||||
{077777777777 + 1, 12, false},
|
||||
{math.MaxInt64, 22, true},
|
||||
{012345670123, 12, true},
|
||||
{01564164, 12, true},
|
||||
{-012345670123, 12, false},
|
||||
{-01564164, 12, false},
|
||||
{-1564164, 30, false},
|
||||
}
|
||||
|
||||
for _, v := range vectors {
|
||||
ok := fitsInOctal(v.width, v.input)
|
||||
if ok != v.ok {
|
||||
t.Errorf("checkOctal(%d, %d): got %v, want %v", v.input, v.width, ok, v.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePAXTime(t *testing.T) {
|
||||
vectors := []struct {
|
||||
in string
|
||||
|
@ -236,6 +294,51 @@ func TestParsePAXTime(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestFormatPAXTime(t *testing.T) {
|
||||
vectors := []struct {
|
||||
sec, nsec int64
|
||||
want string
|
||||
}{
|
||||
{1350244992, 0, "1350244992"},
|
||||
{1350244992, 300000000, "1350244992.3"},
|
||||
{1350244992, 23960100, "1350244992.0239601"},
|
||||
{1350244992, 23960108, "1350244992.023960108"},
|
||||
{+1, +1E9 - 1E0, "1.999999999"},
|
||||
{+1, +1E9 - 1E3, "1.999999"},
|
||||
{+1, +1E9 - 1E6, "1.999"},
|
||||
{+1, +0E0 - 0E0, "1"},
|
||||
{+1, +1E6 - 0E0, "1.001"},
|
||||
{+1, +1E3 - 0E0, "1.000001"},
|
||||
{+1, +1E0 - 0E0, "1.000000001"},
|
||||
{0, 1E9 - 1E0, "0.999999999"},
|
||||
{0, 1E9 - 1E3, "0.999999"},
|
||||
{0, 1E9 - 1E6, "0.999"},
|
||||
{0, 0E0, "0"},
|
||||
{0, 1E6 + 0E0, "0.001"},
|
||||
{0, 1E3 + 0E0, "0.000001"},
|
||||
{0, 1E0 + 0E0, "0.000000001"},
|
||||
{-1, -1E9 + 1E0, "-1.999999999"},
|
||||
{-1, -1E9 + 1E3, "-1.999999"},
|
||||
{-1, -1E9 + 1E6, "-1.999"},
|
||||
{-1, -0E0 + 0E0, "-1"},
|
||||
{-1, -1E6 + 0E0, "-1.001"},
|
||||
{-1, -1E3 + 0E0, "-1.000001"},
|
||||
{-1, -1E0 + 0E0, "-1.000000001"},
|
||||
{-1350244992, 0, "-1350244992"},
|
||||
{-1350244992, -300000000, "-1350244992.3"},
|
||||
{-1350244992, -23960100, "-1350244992.0239601"},
|
||||
{-1350244992, -23960108, "-1350244992.023960108"},
|
||||
}
|
||||
|
||||
for _, v := range vectors {
|
||||
got := formatPAXTime(time.Unix(v.sec, v.nsec))
|
||||
if got != v.want {
|
||||
t.Errorf("formatPAXTime(%ds, %dns): got %q, want %q",
|
||||
v.sec, v.nsec, got, v.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePAXRecord(t *testing.T) {
|
||||
medName := strings.Repeat("CD", 50)
|
||||
longName := strings.Repeat("AB", 100)
|
||||
|
@ -256,7 +359,7 @@ func TestParsePAXRecord(t *testing.T) {
|
|||
{"18 foo=b=\nar=\n==\x00\n", "", "foo", "b=\nar=\n==\x00", true},
|
||||
{"27 foo=hello9 foo=ba\nworld\n", "", "foo", "hello9 foo=ba\nworld", true},
|
||||
{"27 ☺☻☹=日a本b語ç\nmeow mix", "meow mix", "☺☻☹", "日a本b語ç", true},
|
||||
{"17 \x00hello=\x00world\n", "", "\x00hello", "\x00world", true},
|
||||
{"17 \x00hello=\x00world\n", "17 \x00hello=\x00world\n", "", "", false},
|
||||
{"1 k=1\n", "1 k=1\n", "", "", false},
|
||||
{"6 k~1\n", "6 k~1\n", "", "", false},
|
||||
{"6_k=1\n", "6_k=1\n", "", "", false},
|
||||
|
@ -296,21 +399,33 @@ func TestFormatPAXRecord(t *testing.T) {
|
|||
inKey string
|
||||
inVal string
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
{"k", "v", "6 k=v\n"},
|
||||
{"path", "/etc/hosts", "19 path=/etc/hosts\n"},
|
||||
{"path", longName, "210 path=" + longName + "\n"},
|
||||
{"path", medName, "110 path=" + medName + "\n"},
|
||||
{"foo", "ba", "9 foo=ba\n"},
|
||||
{"foo", "bar", "11 foo=bar\n"},
|
||||
{"foo", "b=\nar=\n==\x00", "18 foo=b=\nar=\n==\x00\n"},
|
||||
{"foo", "hello9 foo=ba\nworld", "27 foo=hello9 foo=ba\nworld\n"},
|
||||
{"☺☻☹", "日a本b語ç", "27 ☺☻☹=日a本b語ç\n"},
|
||||
{"\x00hello", "\x00world", "17 \x00hello=\x00world\n"},
|
||||
{"k", "v", "6 k=v\n", true},
|
||||
{"path", "/etc/hosts", "19 path=/etc/hosts\n", true},
|
||||
{"path", longName, "210 path=" + longName + "\n", true},
|
||||
{"path", medName, "110 path=" + medName + "\n", true},
|
||||
{"foo", "ba", "9 foo=ba\n", true},
|
||||
{"foo", "bar", "11 foo=bar\n", true},
|
||||
{"foo", "b=\nar=\n==\x00", "18 foo=b=\nar=\n==\x00\n", true},
|
||||
{"foo", "hello9 foo=ba\nworld", "27 foo=hello9 foo=ba\nworld\n", true},
|
||||
{"☺☻☹", "日a本b語ç", "27 ☺☻☹=日a本b語ç\n", true},
|
||||
{"xhello", "\x00world", "17 xhello=\x00world\n", true},
|
||||
{"path", "null\x00", "", false},
|
||||
{"null\x00", "value", "", false},
|
||||
{paxSchilyXattr + "key", "null\x00", "26 SCHILY.xattr.key=null\x00\n", true},
|
||||
}
|
||||
|
||||
for _, v := range vectors {
|
||||
got := formatPAXRecord(v.inKey, v.inVal)
|
||||
got, err := formatPAXRecord(v.inKey, v.inVal)
|
||||
ok := (err == nil)
|
||||
if ok != v.ok {
|
||||
if v.ok {
|
||||
t.Errorf("formatPAXRecord(%q, %q): got format failure, want success", v.inKey, v.inVal)
|
||||
} else {
|
||||
t.Errorf("formatPAXRecord(%q, %q): got format success, want failure", v.inKey, v.inVal)
|
||||
}
|
||||
}
|
||||
if got != v.want {
|
||||
t.Errorf("formatPAXRecord(%q, %q): got %q, want %q",
|
||||
v.inKey, v.inVal, got, v.want)
|
||||
|
|
|
@ -6,8 +6,12 @@ package tar
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"internal/testenv"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
@ -17,6 +21,193 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
type testError struct{ error }
|
||||
|
||||
type fileOps []interface{} // []T where T is (string | int64)
|
||||
|
||||
// testFile is an io.ReadWriteSeeker where the IO operations performed
|
||||
// on it must match the list of operations in ops.
|
||||
type testFile struct {
|
||||
ops fileOps
|
||||
pos int64
|
||||
}
|
||||
|
||||
func (f *testFile) Read(b []byte) (int, error) {
|
||||
if len(b) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if len(f.ops) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
s, ok := f.ops[0].(string)
|
||||
if !ok {
|
||||
return 0, errors.New("unexpected Read operation")
|
||||
}
|
||||
|
||||
n := copy(b, s)
|
||||
if len(s) > n {
|
||||
f.ops[0] = s[n:]
|
||||
} else {
|
||||
f.ops = f.ops[1:]
|
||||
}
|
||||
f.pos += int64(len(b))
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (f *testFile) Write(b []byte) (int, error) {
|
||||
if len(b) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if len(f.ops) == 0 {
|
||||
return 0, errors.New("unexpected Write operation")
|
||||
}
|
||||
s, ok := f.ops[0].(string)
|
||||
if !ok {
|
||||
return 0, errors.New("unexpected Write operation")
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(s, string(b)) {
|
||||
return 0, testError{fmt.Errorf("got Write(%q), want Write(%q)", b, s)}
|
||||
}
|
||||
if len(s) > len(b) {
|
||||
f.ops[0] = s[len(b):]
|
||||
} else {
|
||||
f.ops = f.ops[1:]
|
||||
}
|
||||
f.pos += int64(len(b))
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
func (f *testFile) Seek(pos int64, whence int) (int64, error) {
|
||||
if pos == 0 && whence == io.SeekCurrent {
|
||||
return f.pos, nil
|
||||
}
|
||||
if len(f.ops) == 0 {
|
||||
return 0, errors.New("unexpected Seek operation")
|
||||
}
|
||||
s, ok := f.ops[0].(int64)
|
||||
if !ok {
|
||||
return 0, errors.New("unexpected Seek operation")
|
||||
}
|
||||
|
||||
if s != pos || whence != io.SeekCurrent {
|
||||
return 0, testError{fmt.Errorf("got Seek(%d, %d), want Seek(%d, %d)", pos, whence, s, io.SeekCurrent)}
|
||||
}
|
||||
f.pos += s
|
||||
f.ops = f.ops[1:]
|
||||
return f.pos, nil
|
||||
}
|
||||
|
||||
func equalSparseEntries(x, y []sparseEntry) bool {
|
||||
return (len(x) == 0 && len(y) == 0) || reflect.DeepEqual(x, y)
|
||||
}
|
||||
|
||||
func TestSparseEntries(t *testing.T) {
|
||||
vectors := []struct {
|
||||
in []sparseEntry
|
||||
size int64
|
||||
|
||||
wantValid bool // Result of validateSparseEntries
|
||||
wantAligned []sparseEntry // Result of alignSparseEntries
|
||||
wantInverted []sparseEntry // Result of invertSparseEntries
|
||||
}{{
|
||||
in: []sparseEntry{}, size: 0,
|
||||
wantValid: true,
|
||||
wantInverted: []sparseEntry{{0, 0}},
|
||||
}, {
|
||||
in: []sparseEntry{}, size: 5000,
|
||||
wantValid: true,
|
||||
wantInverted: []sparseEntry{{0, 5000}},
|
||||
}, {
|
||||
in: []sparseEntry{{0, 5000}}, size: 5000,
|
||||
wantValid: true,
|
||||
wantAligned: []sparseEntry{{0, 5000}},
|
||||
wantInverted: []sparseEntry{{5000, 0}},
|
||||
}, {
|
||||
in: []sparseEntry{{1000, 4000}}, size: 5000,
|
||||
wantValid: true,
|
||||
wantAligned: []sparseEntry{{1024, 3976}},
|
||||
wantInverted: []sparseEntry{{0, 1000}, {5000, 0}},
|
||||
}, {
|
||||
in: []sparseEntry{{0, 3000}}, size: 5000,
|
||||
wantValid: true,
|
||||
wantAligned: []sparseEntry{{0, 2560}},
|
||||
wantInverted: []sparseEntry{{3000, 2000}},
|
||||
}, {
|
||||
in: []sparseEntry{{3000, 2000}}, size: 5000,
|
||||
wantValid: true,
|
||||
wantAligned: []sparseEntry{{3072, 1928}},
|
||||
wantInverted: []sparseEntry{{0, 3000}, {5000, 0}},
|
||||
}, {
|
||||
in: []sparseEntry{{2000, 2000}}, size: 5000,
|
||||
wantValid: true,
|
||||
wantAligned: []sparseEntry{{2048, 1536}},
|
||||
wantInverted: []sparseEntry{{0, 2000}, {4000, 1000}},
|
||||
}, {
|
||||
in: []sparseEntry{{0, 2000}, {8000, 2000}}, size: 10000,
|
||||
wantValid: true,
|
||||
wantAligned: []sparseEntry{{0, 1536}, {8192, 1808}},
|
||||
wantInverted: []sparseEntry{{2000, 6000}, {10000, 0}},
|
||||
}, {
|
||||
in: []sparseEntry{{0, 2000}, {2000, 2000}, {4000, 0}, {4000, 3000}, {7000, 1000}, {8000, 0}, {8000, 2000}}, size: 10000,
|
||||
wantValid: true,
|
||||
wantAligned: []sparseEntry{{0, 1536}, {2048, 1536}, {4096, 2560}, {7168, 512}, {8192, 1808}},
|
||||
wantInverted: []sparseEntry{{10000, 0}},
|
||||
}, {
|
||||
in: []sparseEntry{{0, 0}, {1000, 0}, {2000, 0}, {3000, 0}, {4000, 0}, {5000, 0}}, size: 5000,
|
||||
wantValid: true,
|
||||
wantInverted: []sparseEntry{{0, 5000}},
|
||||
}, {
|
||||
in: []sparseEntry{{1, 0}}, size: 0,
|
||||
wantValid: false,
|
||||
}, {
|
||||
in: []sparseEntry{{-1, 0}}, size: 100,
|
||||
wantValid: false,
|
||||
}, {
|
||||
in: []sparseEntry{{0, -1}}, size: 100,
|
||||
wantValid: false,
|
||||
}, {
|
||||
in: []sparseEntry{{0, 0}}, size: -100,
|
||||
wantValid: false,
|
||||
}, {
|
||||
in: []sparseEntry{{math.MaxInt64, 3}, {6, -5}}, size: 35,
|
||||
wantValid: false,
|
||||
}, {
|
||||
in: []sparseEntry{{1, 3}, {6, -5}}, size: 35,
|
||||
wantValid: false,
|
||||
}, {
|
||||
in: []sparseEntry{{math.MaxInt64, math.MaxInt64}}, size: math.MaxInt64,
|
||||
wantValid: false,
|
||||
}, {
|
||||
in: []sparseEntry{{3, 3}}, size: 5,
|
||||
wantValid: false,
|
||||
}, {
|
||||
in: []sparseEntry{{2, 0}, {1, 0}, {0, 0}}, size: 3,
|
||||
wantValid: false,
|
||||
}, {
|
||||
in: []sparseEntry{{1, 3}, {2, 2}}, size: 10,
|
||||
wantValid: false,
|
||||
}}
|
||||
|
||||
for i, v := range vectors {
|
||||
gotValid := validateSparseEntries(v.in, v.size)
|
||||
if gotValid != v.wantValid {
|
||||
t.Errorf("test %d, validateSparseEntries() = %v, want %v", i, gotValid, v.wantValid)
|
||||
}
|
||||
if !v.wantValid {
|
||||
continue
|
||||
}
|
||||
gotAligned := alignSparseEntries(append([]sparseEntry{}, v.in...), v.size)
|
||||
if !equalSparseEntries(gotAligned, v.wantAligned) {
|
||||
t.Errorf("test %d, alignSparseEntries():\ngot %v\nwant %v", i, gotAligned, v.wantAligned)
|
||||
}
|
||||
gotInverted := invertSparseEntries(append([]sparseEntry{}, v.in...), v.size)
|
||||
if !equalSparseEntries(gotInverted, v.wantInverted) {
|
||||
t.Errorf("test %d, inverseSparseEntries():\ngot %v\nwant %v", i, gotInverted, v.wantInverted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileInfoHeader(t *testing.T) {
|
||||
fi, err := os.Stat("testdata/small.txt")
|
||||
if err != nil {
|
||||
|
@ -109,15 +300,12 @@ func TestRoundTrip(t *testing.T) {
|
|||
var b bytes.Buffer
|
||||
tw := NewWriter(&b)
|
||||
hdr := &Header{
|
||||
Name: "file.txt",
|
||||
Uid: 1 << 21, // too big for 8 octal digits
|
||||
Size: int64(len(data)),
|
||||
// AddDate to strip monotonic clock reading,
|
||||
// and Round to discard sub-second precision,
|
||||
// both of which are not included in the tar header
|
||||
// and would otherwise break the round-trip check
|
||||
// below.
|
||||
ModTime: time.Now().AddDate(0, 0, 0).Round(1 * time.Second),
|
||||
Name: "file.txt",
|
||||
Uid: 1 << 21, // Too big for 8 octal digits
|
||||
Size: int64(len(data)),
|
||||
ModTime: time.Now().Round(time.Second),
|
||||
PAXRecords: map[string]string{"uid": "2097152"},
|
||||
Format: FormatPAX,
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
t.Fatalf("tw.WriteHeader: %v", err)
|
||||
|
@ -329,3 +517,338 @@ func TestHeaderRoundTrip(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderAllowedFormats(t *testing.T) {
|
||||
vectors := []struct {
|
||||
header *Header // Input header
|
||||
paxHdrs map[string]string // Expected PAX headers that may be needed
|
||||
formats Format // Expected formats that can encode the header
|
||||
}{{
|
||||
header: &Header{},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Size: 077777777777},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Size: 077777777777, Format: FormatUSTAR},
|
||||
formats: FormatUSTAR,
|
||||
}, {
|
||||
header: &Header{Size: 077777777777, Format: FormatPAX},
|
||||
formats: FormatUSTAR | FormatPAX,
|
||||
}, {
|
||||
header: &Header{Size: 077777777777, Format: FormatGNU},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{Size: 077777777777 + 1},
|
||||
paxHdrs: map[string]string{paxSize: "8589934592"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Size: 077777777777 + 1, Format: FormatPAX},
|
||||
paxHdrs: map[string]string{paxSize: "8589934592"},
|
||||
formats: FormatPAX,
|
||||
}, {
|
||||
header: &Header{Size: 077777777777 + 1, Format: FormatGNU},
|
||||
paxHdrs: map[string]string{paxSize: "8589934592"},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{Mode: 07777777},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Mode: 07777777 + 1},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{Devmajor: -123},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{Devmajor: 1<<56 - 1},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{Devmajor: 1 << 56},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{Devmajor: -1 << 56},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{Devmajor: -1<<56 - 1},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{Name: "用戶名", Devmajor: -1 << 56},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{Size: math.MaxInt64},
|
||||
paxHdrs: map[string]string{paxSize: "9223372036854775807"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Size: math.MinInt64},
|
||||
paxHdrs: map[string]string{paxSize: "-9223372036854775808"},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{Uname: "0123456789abcdef0123456789abcdef"},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Uname: "0123456789abcdef0123456789abcdefx"},
|
||||
paxHdrs: map[string]string{paxUname: "0123456789abcdef0123456789abcdefx"},
|
||||
formats: FormatPAX,
|
||||
}, {
|
||||
header: &Header{Name: "foobar"},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Name: strings.Repeat("a", nameSize)},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Name: strings.Repeat("a", nameSize+1)},
|
||||
paxHdrs: map[string]string{paxPath: strings.Repeat("a", nameSize+1)},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Linkname: "用戶名"},
|
||||
paxHdrs: map[string]string{paxLinkpath: "用戶名"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Linkname: strings.Repeat("用戶名\x00", nameSize)},
|
||||
paxHdrs: map[string]string{paxLinkpath: strings.Repeat("用戶名\x00", nameSize)},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{Linkname: "\x00hello"},
|
||||
paxHdrs: map[string]string{paxLinkpath: "\x00hello"},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{Uid: 07777777},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Uid: 07777777 + 1},
|
||||
paxHdrs: map[string]string{paxUid: "2097152"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Xattrs: nil},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Xattrs: map[string]string{"foo": "bar"}},
|
||||
paxHdrs: map[string]string{paxSchilyXattr + "foo": "bar"},
|
||||
formats: FormatPAX,
|
||||
}, {
|
||||
header: &Header{Xattrs: map[string]string{"foo": "bar"}, Format: FormatGNU},
|
||||
paxHdrs: map[string]string{paxSchilyXattr + "foo": "bar"},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{Xattrs: map[string]string{"用戶名": "\x00hello"}},
|
||||
paxHdrs: map[string]string{paxSchilyXattr + "用戶名": "\x00hello"},
|
||||
formats: FormatPAX,
|
||||
}, {
|
||||
header: &Header{Xattrs: map[string]string{"foo=bar": "baz"}},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{Xattrs: map[string]string{"foo": ""}},
|
||||
paxHdrs: map[string]string{paxSchilyXattr + "foo": ""},
|
||||
formats: FormatPAX,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(0, 0)},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(077777777777, 0)},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(077777777777+1, 0)},
|
||||
paxHdrs: map[string]string{paxMtime: "8589934592"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(math.MaxInt64, 0)},
|
||||
paxHdrs: map[string]string{paxMtime: "9223372036854775807"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(math.MaxInt64, 0), Format: FormatUSTAR},
|
||||
paxHdrs: map[string]string{paxMtime: "9223372036854775807"},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(-1, 0)},
|
||||
paxHdrs: map[string]string{paxMtime: "-1"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(1, 500)},
|
||||
paxHdrs: map[string]string{paxMtime: "1.0000005"},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(1, 0)},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(1, 0), Format: FormatPAX},
|
||||
formats: FormatUSTAR | FormatPAX,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(1, 500), Format: FormatUSTAR},
|
||||
paxHdrs: map[string]string{paxMtime: "1.0000005"},
|
||||
formats: FormatUSTAR,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(1, 500), Format: FormatPAX},
|
||||
paxHdrs: map[string]string{paxMtime: "1.0000005"},
|
||||
formats: FormatPAX,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(1, 500), Format: FormatGNU},
|
||||
paxHdrs: map[string]string{paxMtime: "1.0000005"},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(-1, 500)},
|
||||
paxHdrs: map[string]string{paxMtime: "-0.9999995"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{ModTime: time.Unix(-1, 500), Format: FormatGNU},
|
||||
paxHdrs: map[string]string{paxMtime: "-0.9999995"},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{AccessTime: time.Unix(0, 0)},
|
||||
paxHdrs: map[string]string{paxAtime: "0"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{AccessTime: time.Unix(0, 0), Format: FormatUSTAR},
|
||||
paxHdrs: map[string]string{paxAtime: "0"},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{AccessTime: time.Unix(0, 0), Format: FormatPAX},
|
||||
paxHdrs: map[string]string{paxAtime: "0"},
|
||||
formats: FormatPAX,
|
||||
}, {
|
||||
header: &Header{AccessTime: time.Unix(0, 0), Format: FormatGNU},
|
||||
paxHdrs: map[string]string{paxAtime: "0"},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{AccessTime: time.Unix(-123, 0)},
|
||||
paxHdrs: map[string]string{paxAtime: "-123"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{AccessTime: time.Unix(-123, 0), Format: FormatPAX},
|
||||
paxHdrs: map[string]string{paxAtime: "-123"},
|
||||
formats: FormatPAX,
|
||||
}, {
|
||||
header: &Header{ChangeTime: time.Unix(123, 456)},
|
||||
paxHdrs: map[string]string{paxCtime: "123.000000456"},
|
||||
formats: FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{ChangeTime: time.Unix(123, 456), Format: FormatUSTAR},
|
||||
paxHdrs: map[string]string{paxCtime: "123.000000456"},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{ChangeTime: time.Unix(123, 456), Format: FormatGNU},
|
||||
paxHdrs: map[string]string{paxCtime: "123.000000456"},
|
||||
formats: FormatGNU,
|
||||
}, {
|
||||
header: &Header{ChangeTime: time.Unix(123, 456), Format: FormatPAX},
|
||||
paxHdrs: map[string]string{paxCtime: "123.000000456"},
|
||||
formats: FormatPAX,
|
||||
}, {
|
||||
header: &Header{Name: "foo/", Typeflag: TypeDir},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}, {
|
||||
header: &Header{Name: "foo/", Typeflag: TypeReg},
|
||||
formats: FormatUnknown,
|
||||
}, {
|
||||
header: &Header{Name: "foo/", Typeflag: TypeSymlink},
|
||||
formats: FormatUSTAR | FormatPAX | FormatGNU,
|
||||
}}
|
||||
|
||||
for i, v := range vectors {
|
||||
formats, paxHdrs, err := v.header.allowedFormats()
|
||||
if formats != v.formats {
|
||||
t.Errorf("test %d, allowedFormats(): got %v, want %v", i, formats, v.formats)
|
||||
}
|
||||
if formats&FormatPAX > 0 && !reflect.DeepEqual(paxHdrs, v.paxHdrs) && !(len(paxHdrs) == 0 && len(v.paxHdrs) == 0) {
|
||||
t.Errorf("test %d, allowedFormats():\ngot %v\nwant %s", i, paxHdrs, v.paxHdrs)
|
||||
}
|
||||
if (formats != FormatUnknown) && (err != nil) {
|
||||
t.Errorf("test %d, unexpected error: %v", i, err)
|
||||
}
|
||||
if (formats == FormatUnknown) && (err == nil) {
|
||||
t.Errorf("test %d, got nil-error, want non-nil error", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Benchmark(b *testing.B) {
|
||||
type file struct {
|
||||
hdr *Header
|
||||
body []byte
|
||||
}
|
||||
|
||||
vectors := []struct {
|
||||
label string
|
||||
files []file
|
||||
}{{
|
||||
"USTAR",
|
||||
[]file{{
|
||||
&Header{Name: "bar", Mode: 0640, Size: int64(3)},
|
||||
[]byte("foo"),
|
||||
}, {
|
||||
&Header{Name: "world", Mode: 0640, Size: int64(5)},
|
||||
[]byte("hello"),
|
||||
}},
|
||||
}, {
|
||||
"GNU",
|
||||
[]file{{
|
||||
&Header{Name: "bar", Mode: 0640, Size: int64(3), Devmajor: -1},
|
||||
[]byte("foo"),
|
||||
}, {
|
||||
&Header{Name: "world", Mode: 0640, Size: int64(5), Devmajor: -1},
|
||||
[]byte("hello"),
|
||||
}},
|
||||
}, {
|
||||
"PAX",
|
||||
[]file{{
|
||||
&Header{Name: "bar", Mode: 0640, Size: int64(3), Xattrs: map[string]string{"foo": "bar"}},
|
||||
[]byte("foo"),
|
||||
}, {
|
||||
&Header{Name: "world", Mode: 0640, Size: int64(5), Xattrs: map[string]string{"foo": "bar"}},
|
||||
[]byte("hello"),
|
||||
}},
|
||||
}}
|
||||
|
||||
b.Run("Writer", func(b *testing.B) {
|
||||
for _, v := range vectors {
|
||||
b.Run(v.label, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Writing to ioutil.Discard because we want to
|
||||
// test purely the writer code and not bring in disk performance into this.
|
||||
tw := NewWriter(ioutil.Discard)
|
||||
for _, file := range v.files {
|
||||
if err := tw.WriteHeader(file.hdr); err != nil {
|
||||
b.Errorf("unexpected WriteHeader error: %v", err)
|
||||
}
|
||||
if _, err := tw.Write(file.body); err != nil {
|
||||
b.Errorf("unexpected Write error: %v", err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
b.Errorf("unexpected Close error: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("Reader", func(b *testing.B) {
|
||||
for _, v := range vectors {
|
||||
var buf bytes.Buffer
|
||||
var r bytes.Reader
|
||||
|
||||
// Write the archive to a byte buffer.
|
||||
tw := NewWriter(&buf)
|
||||
for _, file := range v.files {
|
||||
tw.WriteHeader(file.hdr)
|
||||
tw.Write(file.body)
|
||||
}
|
||||
tw.Close()
|
||||
b.Run(v.label, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
// Read from the byte buffer.
|
||||
for i := 0; i < b.N; i++ {
|
||||
r.Reset(buf.Bytes())
|
||||
tr := NewReader(&r)
|
||||
if _, err := tr.Next(); err != nil {
|
||||
b.Errorf("unexpected Next error: %v", err)
|
||||
}
|
||||
if _, err := io.Copy(ioutil.Discard, tr); err != nil {
|
||||
b.Errorf("unexpected Copy error : %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -4,255 +4,391 @@
|
|||
|
||||
package tar
|
||||
|
||||
// TODO(dsymonds):
|
||||
// - catch more errors (no first header, etc.)
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrWriteTooLong = errors.New("archive/tar: write too long")
|
||||
ErrFieldTooLong = errors.New("archive/tar: header field too long")
|
||||
ErrWriteAfterClose = errors.New("archive/tar: write after close")
|
||||
errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values")
|
||||
)
|
||||
|
||||
// A Writer provides sequential writing of a tar archive in POSIX.1 format.
|
||||
// A tar archive consists of a sequence of files.
|
||||
// Call WriteHeader to begin a new file, and then call Write to supply that file's data,
|
||||
// writing at most hdr.Size bytes in total.
|
||||
// Writer provides sequential writing of a tar archive.
|
||||
// Write.WriteHeader begins a new file with the provided Header,
|
||||
// and then Writer can be treated as an io.Writer to supply that file's data.
|
||||
type Writer struct {
|
||||
w io.Writer
|
||||
err error
|
||||
nb int64 // number of unwritten bytes for current file entry
|
||||
pad int64 // amount of padding to write after current file entry
|
||||
closed bool
|
||||
usedBinary bool // whether the binary numeric field extension was used
|
||||
preferPax bool // use PAX header instead of binary numeric header
|
||||
hdrBuff block // buffer to use in writeHeader when writing a regular header
|
||||
paxHdrBuff block // buffer to use in writeHeader when writing a PAX header
|
||||
w io.Writer
|
||||
pad int64 // Amount of padding to write after current file entry
|
||||
curr fileWriter // Writer for current file entry
|
||||
hdr Header // Shallow copy of Header that is safe for mutations
|
||||
blk block // Buffer to use as temporary local storage
|
||||
|
||||
// err is a persistent error.
|
||||
// It is only the responsibility of every exported method of Writer to
|
||||
// ensure that this error is sticky.
|
||||
err error
|
||||
}
|
||||
|
||||
// NewWriter creates a new Writer writing to w.
|
||||
func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
|
||||
func NewWriter(w io.Writer) *Writer {
|
||||
return &Writer{w: w, curr: ®FileWriter{w, 0}}
|
||||
}
|
||||
|
||||
// Flush finishes writing the current file (optional).
|
||||
type fileWriter interface {
|
||||
io.Writer
|
||||
fileState
|
||||
|
||||
ReadFrom(io.Reader) (int64, error)
|
||||
}
|
||||
|
||||
// Flush finishes writing the current file's block padding.
|
||||
// The current file must be fully written before Flush can be called.
|
||||
//
|
||||
// This is unnecessary as the next call to WriteHeader or Close
|
||||
// will implicitly flush out the file's padding.
|
||||
func (tw *Writer) Flush() error {
|
||||
if tw.nb > 0 {
|
||||
tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb)
|
||||
return tw.err
|
||||
}
|
||||
|
||||
n := tw.nb + tw.pad
|
||||
for n > 0 && tw.err == nil {
|
||||
nr := n
|
||||
if nr > blockSize {
|
||||
nr = blockSize
|
||||
}
|
||||
var nw int
|
||||
nw, tw.err = tw.w.Write(zeroBlock[0:nr])
|
||||
n -= int64(nw)
|
||||
}
|
||||
tw.nb = 0
|
||||
tw.pad = 0
|
||||
return tw.err
|
||||
}
|
||||
|
||||
var (
|
||||
minTime = time.Unix(0, 0)
|
||||
// There is room for 11 octal digits (33 bits) of mtime.
|
||||
maxTime = minTime.Add((1<<33 - 1) * time.Second)
|
||||
)
|
||||
|
||||
// WriteHeader writes hdr and prepares to accept the file's contents.
|
||||
// WriteHeader calls Flush if it is not the first header.
|
||||
// Calling after a Close will return ErrWriteAfterClose.
|
||||
func (tw *Writer) WriteHeader(hdr *Header) error {
|
||||
return tw.writeHeader(hdr, true)
|
||||
}
|
||||
|
||||
// WriteHeader writes hdr and prepares to accept the file's contents.
|
||||
// WriteHeader calls Flush if it is not the first header.
|
||||
// Calling after a Close will return ErrWriteAfterClose.
|
||||
// As this method is called internally by writePax header to allow it to
|
||||
// suppress writing the pax header.
|
||||
func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
|
||||
if tw.closed {
|
||||
return ErrWriteAfterClose
|
||||
}
|
||||
if tw.err == nil {
|
||||
tw.Flush()
|
||||
}
|
||||
if tw.err != nil {
|
||||
return tw.err
|
||||
}
|
||||
|
||||
// a map to hold pax header records, if any are needed
|
||||
paxHeaders := make(map[string]string)
|
||||
|
||||
// TODO(dsnet): we might want to use PAX headers for
|
||||
// subsecond time resolution, but for now let's just capture
|
||||
// too long fields or non ascii characters
|
||||
|
||||
// We need to select which scratch buffer to use carefully,
|
||||
// since this method is called recursively to write PAX headers.
|
||||
// If allowPax is true, this is the non-recursive call, and we will use hdrBuff.
|
||||
// If allowPax is false, we are being called by writePAXHeader, and hdrBuff is
|
||||
// already being used by the non-recursive call, so we must use paxHdrBuff.
|
||||
header := &tw.hdrBuff
|
||||
if !allowPax {
|
||||
header = &tw.paxHdrBuff
|
||||
if nb := tw.curr.LogicalRemaining(); nb > 0 {
|
||||
return fmt.Errorf("archive/tar: missed writing %d bytes", nb)
|
||||
}
|
||||
copy(header[:], zeroBlock[:])
|
||||
|
||||
// Wrappers around formatter that automatically sets paxHeaders if the
|
||||
// argument extends beyond the capacity of the input byte slice.
|
||||
var f formatter
|
||||
var formatString = func(b []byte, s string, paxKeyword string) {
|
||||
needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s)
|
||||
if needsPaxHeader {
|
||||
paxHeaders[paxKeyword] = s
|
||||
}
|
||||
|
||||
// Write string in a best-effort manner to satisfy readers that expect
|
||||
// the field to be non-empty.
|
||||
s = toASCII(s)
|
||||
if len(s) > len(b) {
|
||||
s = s[:len(b)]
|
||||
}
|
||||
f.formatString(b, s) // Should never error
|
||||
}
|
||||
var formatNumeric = func(b []byte, x int64, paxKeyword string) {
|
||||
// Try octal first.
|
||||
s := strconv.FormatInt(x, 8)
|
||||
if len(s) < len(b) {
|
||||
f.formatOctal(b, x)
|
||||
return
|
||||
}
|
||||
|
||||
// If it is too long for octal, and PAX is preferred, use a PAX header.
|
||||
if paxKeyword != paxNone && tw.preferPax {
|
||||
f.formatOctal(b, 0)
|
||||
s := strconv.FormatInt(x, 10)
|
||||
paxHeaders[paxKeyword] = s
|
||||
return
|
||||
}
|
||||
|
||||
tw.usedBinary = true
|
||||
f.formatNumeric(b, x)
|
||||
}
|
||||
|
||||
// Handle out of range ModTime carefully.
|
||||
var modTime int64
|
||||
if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
|
||||
modTime = hdr.ModTime.Unix()
|
||||
}
|
||||
|
||||
v7 := header.V7()
|
||||
formatString(v7.Name(), hdr.Name, paxPath)
|
||||
// TODO(dsnet): The GNU format permits the mode field to be encoded in
|
||||
// base-256 format. Thus, we can use formatNumeric instead of formatOctal.
|
||||
f.formatOctal(v7.Mode(), hdr.Mode)
|
||||
formatNumeric(v7.UID(), int64(hdr.Uid), paxUid)
|
||||
formatNumeric(v7.GID(), int64(hdr.Gid), paxGid)
|
||||
formatNumeric(v7.Size(), hdr.Size, paxSize)
|
||||
// TODO(dsnet): Consider using PAX for finer time granularity.
|
||||
formatNumeric(v7.ModTime(), modTime, paxNone)
|
||||
v7.TypeFlag()[0] = hdr.Typeflag
|
||||
formatString(v7.LinkName(), hdr.Linkname, paxLinkpath)
|
||||
|
||||
ustar := header.USTAR()
|
||||
formatString(ustar.UserName(), hdr.Uname, paxUname)
|
||||
formatString(ustar.GroupName(), hdr.Gname, paxGname)
|
||||
formatNumeric(ustar.DevMajor(), hdr.Devmajor, paxNone)
|
||||
formatNumeric(ustar.DevMinor(), hdr.Devminor, paxNone)
|
||||
|
||||
// TODO(dsnet): The logic surrounding the prefix field is broken when trying
|
||||
// to encode the header as GNU format. The challenge with the current logic
|
||||
// is that we are unsure what format we are using at any given moment until
|
||||
// we have processed *all* of the fields. The problem is that by the time
|
||||
// all fields have been processed, some work has already been done to handle
|
||||
// each field under the assumption that it is for one given format or
|
||||
// another. In some situations, this causes the Writer to be confused and
|
||||
// encode a prefix field when the format being used is GNU. Thus, producing
|
||||
// an invalid tar file.
|
||||
//
|
||||
// As a short-term fix, we disable the logic to use the prefix field, which
|
||||
// will force the badly generated GNU files to become encoded as being
|
||||
// the PAX format.
|
||||
//
|
||||
// As an alternative fix, we could hard-code preferPax to be true. However,
|
||||
// this is problematic for the following reasons:
|
||||
// * The preferPax functionality is not tested at all.
|
||||
// * This can result in headers that try to use both the GNU and PAX
|
||||
// features at the same time, which is also wrong.
|
||||
//
|
||||
// The proper fix for this is to use a two-pass method:
|
||||
// * The first pass simply determines what set of formats can possibly
|
||||
// encode the given header.
|
||||
// * The second pass actually encodes the header as that given format
|
||||
// without worrying about violating the format.
|
||||
//
|
||||
// See the following:
|
||||
// https://golang.org/issue/12594
|
||||
// https://golang.org/issue/17630
|
||||
// https://golang.org/issue/9683
|
||||
const usePrefix = false
|
||||
|
||||
// try to use a ustar header when only the name is too long
|
||||
_, paxPathUsed := paxHeaders[paxPath]
|
||||
if usePrefix && !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed {
|
||||
prefix, suffix, ok := splitUSTARPath(hdr.Name)
|
||||
if ok {
|
||||
// Since we can encode in USTAR format, disable PAX header.
|
||||
delete(paxHeaders, paxPath)
|
||||
|
||||
// Update the path fields
|
||||
formatString(v7.Name(), suffix, paxNone)
|
||||
formatString(ustar.Prefix(), prefix, paxNone)
|
||||
}
|
||||
}
|
||||
|
||||
if tw.usedBinary {
|
||||
header.SetFormat(formatGNU)
|
||||
} else {
|
||||
header.SetFormat(formatUSTAR)
|
||||
}
|
||||
|
||||
// Check if there were any formatting errors.
|
||||
if f.err != nil {
|
||||
tw.err = f.err
|
||||
if _, tw.err = tw.w.Write(zeroBlock[:tw.pad]); tw.err != nil {
|
||||
return tw.err
|
||||
}
|
||||
tw.pad = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
if allowPax {
|
||||
for k, v := range hdr.Xattrs {
|
||||
paxHeaders[paxXattr+k] = v
|
||||
// WriteHeader writes hdr and prepares to accept the file's contents.
|
||||
// The Header.Size determines how many bytes can be written for the next file.
|
||||
// If the current file is not fully written, then this returns an error.
|
||||
// This implicitly flushes any padding necessary before writing the header.
|
||||
func (tw *Writer) WriteHeader(hdr *Header) error {
|
||||
if err := tw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
tw.hdr = *hdr // Shallow copy of Header
|
||||
|
||||
// Round ModTime and ignore AccessTime and ChangeTime unless
|
||||
// the format is explicitly chosen.
|
||||
// This ensures nominal usage of WriteHeader (without specifying the format)
|
||||
// does not always result in the PAX format being chosen, which
|
||||
// causes a 1KiB increase to every header.
|
||||
if tw.hdr.Format == FormatUnknown {
|
||||
tw.hdr.ModTime = tw.hdr.ModTime.Round(time.Second)
|
||||
tw.hdr.AccessTime = time.Time{}
|
||||
tw.hdr.ChangeTime = time.Time{}
|
||||
}
|
||||
|
||||
allowedFormats, paxHdrs, err := tw.hdr.allowedFormats()
|
||||
switch {
|
||||
case allowedFormats.has(FormatUSTAR):
|
||||
tw.err = tw.writeUSTARHeader(&tw.hdr)
|
||||
return tw.err
|
||||
case allowedFormats.has(FormatPAX):
|
||||
tw.err = tw.writePAXHeader(&tw.hdr, paxHdrs)
|
||||
return tw.err
|
||||
case allowedFormats.has(FormatGNU):
|
||||
tw.err = tw.writeGNUHeader(&tw.hdr)
|
||||
return tw.err
|
||||
default:
|
||||
return err // Non-fatal error
|
||||
}
|
||||
}
|
||||
|
||||
func (tw *Writer) writeUSTARHeader(hdr *Header) error {
|
||||
// Check if we can use USTAR prefix/suffix splitting.
|
||||
var namePrefix string
|
||||
if prefix, suffix, ok := splitUSTARPath(hdr.Name); ok {
|
||||
namePrefix, hdr.Name = prefix, suffix
|
||||
}
|
||||
|
||||
// Pack the main header.
|
||||
var f formatter
|
||||
blk := tw.templateV7Plus(hdr, f.formatString, f.formatOctal)
|
||||
f.formatString(blk.USTAR().Prefix(), namePrefix)
|
||||
blk.SetFormat(FormatUSTAR)
|
||||
if f.err != nil {
|
||||
return f.err // Should never happen since header is validated
|
||||
}
|
||||
return tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag)
|
||||
}
|
||||
|
||||
func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error {
|
||||
realName, realSize := hdr.Name, hdr.Size
|
||||
|
||||
// TODO(dsnet): Re-enable this when adding sparse support.
|
||||
// See https://golang.org/issue/22735
|
||||
/*
|
||||
// Handle sparse files.
|
||||
var spd sparseDatas
|
||||
var spb []byte
|
||||
if len(hdr.SparseHoles) > 0 {
|
||||
sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map
|
||||
sph = alignSparseEntries(sph, hdr.Size)
|
||||
spd = invertSparseEntries(sph, hdr.Size)
|
||||
|
||||
// Format the sparse map.
|
||||
hdr.Size = 0 // Replace with encoded size
|
||||
spb = append(strconv.AppendInt(spb, int64(len(spd)), 10), '\n')
|
||||
for _, s := range spd {
|
||||
hdr.Size += s.Length
|
||||
spb = append(strconv.AppendInt(spb, s.Offset, 10), '\n')
|
||||
spb = append(strconv.AppendInt(spb, s.Length, 10), '\n')
|
||||
}
|
||||
pad := blockPadding(int64(len(spb)))
|
||||
spb = append(spb, zeroBlock[:pad]...)
|
||||
hdr.Size += int64(len(spb)) // Accounts for encoded sparse map
|
||||
|
||||
// Add and modify appropriate PAX records.
|
||||
dir, file := path.Split(realName)
|
||||
hdr.Name = path.Join(dir, "GNUSparseFile.0", file)
|
||||
paxHdrs[paxGNUSparseMajor] = "1"
|
||||
paxHdrs[paxGNUSparseMinor] = "0"
|
||||
paxHdrs[paxGNUSparseName] = realName
|
||||
paxHdrs[paxGNUSparseRealSize] = strconv.FormatInt(realSize, 10)
|
||||
paxHdrs[paxSize] = strconv.FormatInt(hdr.Size, 10)
|
||||
delete(paxHdrs, paxPath) // Recorded by paxGNUSparseName
|
||||
}
|
||||
*/
|
||||
_ = realSize
|
||||
|
||||
// Write PAX records to the output.
|
||||
isGlobal := hdr.Typeflag == TypeXGlobalHeader
|
||||
if len(paxHdrs) > 0 || isGlobal {
|
||||
// Sort keys for deterministic ordering.
|
||||
var keys []string
|
||||
for k := range paxHdrs {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
// Write each record to a buffer.
|
||||
var buf bytes.Buffer
|
||||
for _, k := range keys {
|
||||
rec, err := formatPAXRecord(k, paxHdrs[k])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
buf.WriteString(rec)
|
||||
}
|
||||
|
||||
// Write the extended header file.
|
||||
var name string
|
||||
var flag byte
|
||||
if isGlobal {
|
||||
name = realName
|
||||
if name == "" {
|
||||
name = "GlobalHead.0.0"
|
||||
}
|
||||
flag = TypeXGlobalHeader
|
||||
} else {
|
||||
dir, file := path.Split(realName)
|
||||
name = path.Join(dir, "PaxHeaders.0", file)
|
||||
flag = TypeXHeader
|
||||
}
|
||||
data := buf.String()
|
||||
if err := tw.writeRawFile(name, data, flag, FormatPAX); err != nil || isGlobal {
|
||||
return err // Global headers return here
|
||||
}
|
||||
}
|
||||
|
||||
if len(paxHeaders) > 0 {
|
||||
if !allowPax {
|
||||
return errInvalidHeader
|
||||
// Pack the main header.
|
||||
var f formatter // Ignore errors since they are expected
|
||||
fmtStr := func(b []byte, s string) { f.formatString(b, toASCII(s)) }
|
||||
blk := tw.templateV7Plus(hdr, fmtStr, f.formatOctal)
|
||||
blk.SetFormat(FormatPAX)
|
||||
if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO(dsnet): Re-enable this when adding sparse support.
|
||||
// See https://golang.org/issue/22735
|
||||
/*
|
||||
// Write the sparse map and setup the sparse writer if necessary.
|
||||
if len(spd) > 0 {
|
||||
// Use tw.curr since the sparse map is accounted for in hdr.Size.
|
||||
if _, err := tw.curr.Write(spb); err != nil {
|
||||
return err
|
||||
}
|
||||
tw.curr = &sparseFileWriter{tw.curr, spd, 0}
|
||||
}
|
||||
if err := tw.writePAXHeader(hdr, paxHeaders); err != nil {
|
||||
*/
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tw *Writer) writeGNUHeader(hdr *Header) error {
|
||||
// Use long-link files if Name or Linkname exceeds the field size.
|
||||
const longName = "././@LongLink"
|
||||
if len(hdr.Name) > nameSize {
|
||||
data := hdr.Name + "\x00"
|
||||
if err := tw.writeRawFile(longName, data, TypeGNULongName, FormatGNU); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(hdr.Linkname) > nameSize {
|
||||
data := hdr.Linkname + "\x00"
|
||||
if err := tw.writeRawFile(longName, data, TypeGNULongLink, FormatGNU); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
tw.nb = hdr.Size
|
||||
tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize
|
||||
|
||||
_, tw.err = tw.w.Write(header[:])
|
||||
return tw.err
|
||||
// Pack the main header.
|
||||
var f formatter // Ignore errors since they are expected
|
||||
var spd sparseDatas
|
||||
var spb []byte
|
||||
blk := tw.templateV7Plus(hdr, f.formatString, f.formatNumeric)
|
||||
if !hdr.AccessTime.IsZero() {
|
||||
f.formatNumeric(blk.GNU().AccessTime(), hdr.AccessTime.Unix())
|
||||
}
|
||||
if !hdr.ChangeTime.IsZero() {
|
||||
f.formatNumeric(blk.GNU().ChangeTime(), hdr.ChangeTime.Unix())
|
||||
}
|
||||
// TODO(dsnet): Re-enable this when adding sparse support.
|
||||
// See https://golang.org/issue/22735
|
||||
/*
|
||||
if hdr.Typeflag == TypeGNUSparse {
|
||||
sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map
|
||||
sph = alignSparseEntries(sph, hdr.Size)
|
||||
spd = invertSparseEntries(sph, hdr.Size)
|
||||
|
||||
// Format the sparse map.
|
||||
formatSPD := func(sp sparseDatas, sa sparseArray) sparseDatas {
|
||||
for i := 0; len(sp) > 0 && i < sa.MaxEntries(); i++ {
|
||||
f.formatNumeric(sa.Entry(i).Offset(), sp[0].Offset)
|
||||
f.formatNumeric(sa.Entry(i).Length(), sp[0].Length)
|
||||
sp = sp[1:]
|
||||
}
|
||||
if len(sp) > 0 {
|
||||
sa.IsExtended()[0] = 1
|
||||
}
|
||||
return sp
|
||||
}
|
||||
sp2 := formatSPD(spd, blk.GNU().Sparse())
|
||||
for len(sp2) > 0 {
|
||||
var spHdr block
|
||||
sp2 = formatSPD(sp2, spHdr.Sparse())
|
||||
spb = append(spb, spHdr[:]...)
|
||||
}
|
||||
|
||||
// Update size fields in the header block.
|
||||
realSize := hdr.Size
|
||||
hdr.Size = 0 // Encoded size; does not account for encoded sparse map
|
||||
for _, s := range spd {
|
||||
hdr.Size += s.Length
|
||||
}
|
||||
copy(blk.V7().Size(), zeroBlock[:]) // Reset field
|
||||
f.formatNumeric(blk.V7().Size(), hdr.Size)
|
||||
f.formatNumeric(blk.GNU().RealSize(), realSize)
|
||||
}
|
||||
*/
|
||||
blk.SetFormat(FormatGNU)
|
||||
if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write the extended sparse map and setup the sparse writer if necessary.
|
||||
if len(spd) > 0 {
|
||||
// Use tw.w since the sparse map is not accounted for in hdr.Size.
|
||||
if _, err := tw.w.Write(spb); err != nil {
|
||||
return err
|
||||
}
|
||||
tw.curr = &sparseFileWriter{tw.curr, spd, 0}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type (
|
||||
stringFormatter func([]byte, string)
|
||||
numberFormatter func([]byte, int64)
|
||||
)
|
||||
|
||||
// templateV7Plus fills out the V7 fields of a block using values from hdr.
|
||||
// It also fills out fields (uname, gname, devmajor, devminor) that are
|
||||
// shared in the USTAR, PAX, and GNU formats using the provided formatters.
|
||||
//
|
||||
// The block returned is only valid until the next call to
|
||||
// templateV7Plus or writeRawFile.
|
||||
func (tw *Writer) templateV7Plus(hdr *Header, fmtStr stringFormatter, fmtNum numberFormatter) *block {
|
||||
tw.blk.Reset()
|
||||
|
||||
modTime := hdr.ModTime
|
||||
if modTime.IsZero() {
|
||||
modTime = time.Unix(0, 0)
|
||||
}
|
||||
|
||||
v7 := tw.blk.V7()
|
||||
v7.TypeFlag()[0] = hdr.Typeflag
|
||||
fmtStr(v7.Name(), hdr.Name)
|
||||
fmtStr(v7.LinkName(), hdr.Linkname)
|
||||
fmtNum(v7.Mode(), hdr.Mode)
|
||||
fmtNum(v7.UID(), int64(hdr.Uid))
|
||||
fmtNum(v7.GID(), int64(hdr.Gid))
|
||||
fmtNum(v7.Size(), hdr.Size)
|
||||
fmtNum(v7.ModTime(), modTime.Unix())
|
||||
|
||||
ustar := tw.blk.USTAR()
|
||||
fmtStr(ustar.UserName(), hdr.Uname)
|
||||
fmtStr(ustar.GroupName(), hdr.Gname)
|
||||
fmtNum(ustar.DevMajor(), hdr.Devmajor)
|
||||
fmtNum(ustar.DevMinor(), hdr.Devminor)
|
||||
|
||||
return &tw.blk
|
||||
}
|
||||
|
||||
// writeRawFile writes a minimal file with the given name and flag type.
|
||||
// It uses format to encode the header format and will write data as the body.
|
||||
// It uses default values for all of the other fields (as BSD and GNU tar does).
|
||||
func (tw *Writer) writeRawFile(name, data string, flag byte, format Format) error {
|
||||
tw.blk.Reset()
|
||||
|
||||
// Best effort for the filename.
|
||||
name = toASCII(name)
|
||||
if len(name) > nameSize {
|
||||
name = name[:nameSize]
|
||||
}
|
||||
name = strings.TrimRight(name, "/")
|
||||
|
||||
var f formatter
|
||||
v7 := tw.blk.V7()
|
||||
v7.TypeFlag()[0] = flag
|
||||
f.formatString(v7.Name(), name)
|
||||
f.formatOctal(v7.Mode(), 0)
|
||||
f.formatOctal(v7.UID(), 0)
|
||||
f.formatOctal(v7.GID(), 0)
|
||||
f.formatOctal(v7.Size(), int64(len(data))) // Must be < 8GiB
|
||||
f.formatOctal(v7.ModTime(), 0)
|
||||
tw.blk.SetFormat(format)
|
||||
if f.err != nil {
|
||||
return f.err // Only occurs if size condition is violated
|
||||
}
|
||||
|
||||
// Write the header and data.
|
||||
if err := tw.writeRawHeader(&tw.blk, int64(len(data)), flag); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := io.WriteString(tw, data)
|
||||
return err
|
||||
}
|
||||
|
||||
// writeRawHeader writes the value of blk, regardless of its value.
|
||||
// It sets up the Writer such that it can accept a file of the given size.
|
||||
// If the flag is a special header-only flag, then the size is treated as zero.
|
||||
func (tw *Writer) writeRawHeader(blk *block, size int64, flag byte) error {
|
||||
if err := tw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.w.Write(blk[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if isHeaderOnlyType(flag) {
|
||||
size = 0
|
||||
}
|
||||
tw.curr = ®FileWriter{tw.w, size}
|
||||
tw.pad = blockPadding(size)
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitUSTARPath splits a path according to USTAR prefix and suffix rules.
|
||||
|
@ -276,95 +412,233 @@ func splitUSTARPath(name string) (prefix, suffix string, ok bool) {
|
|||
return name[:i], name[i+1:], true
|
||||
}
|
||||
|
||||
// writePaxHeader writes an extended pax header to the
|
||||
// archive.
|
||||
func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error {
|
||||
// Prepare extended header
|
||||
ext := new(Header)
|
||||
ext.Typeflag = TypeXHeader
|
||||
// Setting ModTime is required for reader parsing to
|
||||
// succeed, and seems harmless enough.
|
||||
ext.ModTime = hdr.ModTime
|
||||
// The spec asks that we namespace our pseudo files
|
||||
// with the current pid. However, this results in differing outputs
|
||||
// for identical inputs. As such, the constant 0 is now used instead.
|
||||
// golang.org/issue/12358
|
||||
dir, file := path.Split(hdr.Name)
|
||||
fullName := path.Join(dir, "PaxHeaders.0", file)
|
||||
|
||||
ascii := toASCII(fullName)
|
||||
if len(ascii) > nameSize {
|
||||
ascii = ascii[:nameSize]
|
||||
}
|
||||
ext.Name = ascii
|
||||
// Construct the body
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Keys are sorted before writing to body to allow deterministic output.
|
||||
keys := make([]string, 0, len(paxHeaders))
|
||||
for k := range paxHeaders {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
for _, k := range keys {
|
||||
fmt.Fprint(&buf, formatPAXRecord(k, paxHeaders[k]))
|
||||
}
|
||||
|
||||
ext.Size = int64(len(buf.Bytes()))
|
||||
if err := tw.writeHeader(ext, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write(buf.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write writes to the current entry in the tar archive.
|
||||
// Write writes to the current file in the tar archive.
|
||||
// Write returns the error ErrWriteTooLong if more than
|
||||
// hdr.Size bytes are written after WriteHeader.
|
||||
func (tw *Writer) Write(b []byte) (n int, err error) {
|
||||
if tw.closed {
|
||||
err = ErrWriteAfterClose
|
||||
return
|
||||
// Header.Size bytes are written after WriteHeader.
|
||||
//
|
||||
// Calling Write on special types like TypeLink, TypeSymlink, TypeChar,
|
||||
// TypeBlock, TypeDir, and TypeFifo returns (0, ErrWriteTooLong) regardless
|
||||
// of what the Header.Size claims.
|
||||
func (tw *Writer) Write(b []byte) (int, error) {
|
||||
if tw.err != nil {
|
||||
return 0, tw.err
|
||||
}
|
||||
overwrite := false
|
||||
if int64(len(b)) > tw.nb {
|
||||
b = b[0:tw.nb]
|
||||
overwrite = true
|
||||
n, err := tw.curr.Write(b)
|
||||
if err != nil && err != ErrWriteTooLong {
|
||||
tw.err = err
|
||||
}
|
||||
n, err = tw.w.Write(b)
|
||||
tw.nb -= int64(n)
|
||||
if err == nil && overwrite {
|
||||
err = ErrWriteTooLong
|
||||
return
|
||||
}
|
||||
tw.err = err
|
||||
return
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes the tar archive, flushing any unwritten
|
||||
// data to the underlying writer.
|
||||
func (tw *Writer) Close() error {
|
||||
if tw.err != nil || tw.closed {
|
||||
return tw.err
|
||||
// readFrom populates the content of the current file by reading from r.
|
||||
// The bytes read must match the number of remaining bytes in the current file.
|
||||
//
|
||||
// If the current file is sparse and r is an io.ReadSeeker,
|
||||
// then readFrom uses Seek to skip past holes defined in Header.SparseHoles,
|
||||
// assuming that skipped regions are all NULs.
|
||||
// This always reads the last byte to ensure r is the right size.
|
||||
//
|
||||
// TODO(dsnet): Re-export this when adding sparse file support.
|
||||
// See https://golang.org/issue/22735
|
||||
func (tw *Writer) readFrom(r io.Reader) (int64, error) {
|
||||
if tw.err != nil {
|
||||
return 0, tw.err
|
||||
}
|
||||
n, err := tw.curr.ReadFrom(r)
|
||||
if err != nil && err != ErrWriteTooLong {
|
||||
tw.err = err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes the tar archive by flushing the padding, and writing the footer.
|
||||
// If the current file (from a prior call to WriteHeader) is not fully written,
|
||||
// then this returns an error.
|
||||
func (tw *Writer) Close() error {
|
||||
if tw.err == ErrWriteAfterClose {
|
||||
return nil
|
||||
}
|
||||
tw.Flush()
|
||||
tw.closed = true
|
||||
if tw.err != nil {
|
||||
return tw.err
|
||||
}
|
||||
|
||||
// trailer: two zero blocks
|
||||
for i := 0; i < 2; i++ {
|
||||
_, tw.err = tw.w.Write(zeroBlock[:])
|
||||
if tw.err != nil {
|
||||
break
|
||||
// Trailer: two zero blocks.
|
||||
err := tw.Flush()
|
||||
for i := 0; i < 2 && err == nil; i++ {
|
||||
_, err = tw.w.Write(zeroBlock[:])
|
||||
}
|
||||
|
||||
// Ensure all future actions are invalid.
|
||||
tw.err = ErrWriteAfterClose
|
||||
return err // Report IO errors
|
||||
}
|
||||
|
||||
// regFileWriter is a fileWriter for writing data to a regular file entry.
|
||||
type regFileWriter struct {
|
||||
w io.Writer // Underlying Writer
|
||||
nb int64 // Number of remaining bytes to write
|
||||
}
|
||||
|
||||
func (fw *regFileWriter) Write(b []byte) (n int, err error) {
|
||||
overwrite := int64(len(b)) > fw.nb
|
||||
if overwrite {
|
||||
b = b[:fw.nb]
|
||||
}
|
||||
if len(b) > 0 {
|
||||
n, err = fw.w.Write(b)
|
||||
fw.nb -= int64(n)
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
return n, err
|
||||
case overwrite:
|
||||
return n, ErrWriteTooLong
|
||||
default:
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (fw *regFileWriter) ReadFrom(r io.Reader) (int64, error) {
|
||||
return io.Copy(struct{ io.Writer }{fw}, r)
|
||||
}
|
||||
|
||||
func (fw regFileWriter) LogicalRemaining() int64 {
|
||||
return fw.nb
|
||||
}
|
||||
func (fw regFileWriter) PhysicalRemaining() int64 {
|
||||
return fw.nb
|
||||
}
|
||||
|
||||
// sparseFileWriter is a fileWriter for writing data to a sparse file entry.
|
||||
type sparseFileWriter struct {
|
||||
fw fileWriter // Underlying fileWriter
|
||||
sp sparseDatas // Normalized list of data fragments
|
||||
pos int64 // Current position in sparse file
|
||||
}
|
||||
|
||||
func (sw *sparseFileWriter) Write(b []byte) (n int, err error) {
|
||||
overwrite := int64(len(b)) > sw.LogicalRemaining()
|
||||
if overwrite {
|
||||
b = b[:sw.LogicalRemaining()]
|
||||
}
|
||||
|
||||
b0 := b
|
||||
endPos := sw.pos + int64(len(b))
|
||||
for endPos > sw.pos && err == nil {
|
||||
var nf int // Bytes written in fragment
|
||||
dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset()
|
||||
if sw.pos < dataStart { // In a hole fragment
|
||||
bf := b[:min(int64(len(b)), dataStart-sw.pos)]
|
||||
nf, err = zeroWriter{}.Write(bf)
|
||||
} else { // In a data fragment
|
||||
bf := b[:min(int64(len(b)), dataEnd-sw.pos)]
|
||||
nf, err = sw.fw.Write(bf)
|
||||
}
|
||||
b = b[nf:]
|
||||
sw.pos += int64(nf)
|
||||
if sw.pos >= dataEnd && len(sw.sp) > 1 {
|
||||
sw.sp = sw.sp[1:] // Ensure last fragment always remains
|
||||
}
|
||||
}
|
||||
return tw.err
|
||||
|
||||
n = len(b0) - len(b)
|
||||
switch {
|
||||
case err == ErrWriteTooLong:
|
||||
return n, errMissData // Not possible; implies bug in validation logic
|
||||
case err != nil:
|
||||
return n, err
|
||||
case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0:
|
||||
return n, errUnrefData // Not possible; implies bug in validation logic
|
||||
case overwrite:
|
||||
return n, ErrWriteTooLong
|
||||
default:
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (sw *sparseFileWriter) ReadFrom(r io.Reader) (n int64, err error) {
|
||||
rs, ok := r.(io.ReadSeeker)
|
||||
if ok {
|
||||
if _, err := rs.Seek(0, io.SeekCurrent); err != nil {
|
||||
ok = false // Not all io.Seeker can really seek
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return io.Copy(struct{ io.Writer }{sw}, r)
|
||||
}
|
||||
|
||||
var readLastByte bool
|
||||
pos0 := sw.pos
|
||||
for sw.LogicalRemaining() > 0 && !readLastByte && err == nil {
|
||||
var nf int64 // Size of fragment
|
||||
dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset()
|
||||
if sw.pos < dataStart { // In a hole fragment
|
||||
nf = dataStart - sw.pos
|
||||
if sw.PhysicalRemaining() == 0 {
|
||||
readLastByte = true
|
||||
nf--
|
||||
}
|
||||
_, err = rs.Seek(nf, io.SeekCurrent)
|
||||
} else { // In a data fragment
|
||||
nf = dataEnd - sw.pos
|
||||
nf, err = io.CopyN(sw.fw, rs, nf)
|
||||
}
|
||||
sw.pos += nf
|
||||
if sw.pos >= dataEnd && len(sw.sp) > 1 {
|
||||
sw.sp = sw.sp[1:] // Ensure last fragment always remains
|
||||
}
|
||||
}
|
||||
|
||||
// If the last fragment is a hole, then seek to 1-byte before EOF, and
|
||||
// read a single byte to ensure the file is the right size.
|
||||
if readLastByte && err == nil {
|
||||
_, err = mustReadFull(rs, []byte{0})
|
||||
sw.pos++
|
||||
}
|
||||
|
||||
n = sw.pos - pos0
|
||||
switch {
|
||||
case err == io.EOF:
|
||||
return n, io.ErrUnexpectedEOF
|
||||
case err == ErrWriteTooLong:
|
||||
return n, errMissData // Not possible; implies bug in validation logic
|
||||
case err != nil:
|
||||
return n, err
|
||||
case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0:
|
||||
return n, errUnrefData // Not possible; implies bug in validation logic
|
||||
default:
|
||||
return n, ensureEOF(rs)
|
||||
}
|
||||
}
|
||||
|
||||
func (sw sparseFileWriter) LogicalRemaining() int64 {
|
||||
return sw.sp[len(sw.sp)-1].endOffset() - sw.pos
|
||||
}
|
||||
func (sw sparseFileWriter) PhysicalRemaining() int64 {
|
||||
return sw.fw.PhysicalRemaining()
|
||||
}
|
||||
|
||||
// zeroWriter may only be written with NULs, otherwise it returns errWriteHole.
|
||||
type zeroWriter struct{}
|
||||
|
||||
func (zeroWriter) Write(b []byte) (int, error) {
|
||||
for i, c := range b {
|
||||
if c != 0 {
|
||||
return i, errWriteHole
|
||||
}
|
||||
}
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
// ensureEOF checks whether r is at EOF, reporting ErrWriteTooLong if not so.
|
||||
func ensureEOF(r io.Reader) error {
|
||||
n, err := tryReadFull(r, []byte{0})
|
||||
switch {
|
||||
case n > 0:
|
||||
return ErrWriteTooLong
|
||||
case err == io.EOF:
|
||||
return nil
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -13,6 +13,7 @@ import (
|
|||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -94,7 +95,7 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
|
|||
|
||||
// The count of files inside a zip is truncated to fit in a uint16.
|
||||
// Gloss over this by reading headers until we encounter
|
||||
// a bad one, and then only report a ErrFormat or UnexpectedEOF if
|
||||
// a bad one, and then only report an ErrFormat or UnexpectedEOF if
|
||||
// the file count modulo 65536 is incorrect.
|
||||
for {
|
||||
f := &File{zip: z, zipr: r, zipsize: size}
|
||||
|
@ -280,52 +281,128 @@ func readDirectoryHeader(f *File, r io.Reader) error {
|
|||
f.Extra = d[filenameLen : filenameLen+extraLen]
|
||||
f.Comment = string(d[filenameLen+extraLen:])
|
||||
|
||||
// Determine the character encoding.
|
||||
utf8Valid1, utf8Require1 := detectUTF8(f.Name)
|
||||
utf8Valid2, utf8Require2 := detectUTF8(f.Comment)
|
||||
switch {
|
||||
case !utf8Valid1 || !utf8Valid2:
|
||||
// Name and Comment definitely not UTF-8.
|
||||
f.NonUTF8 = true
|
||||
case !utf8Require1 && !utf8Require2:
|
||||
// Name and Comment use only single-byte runes that overlap with UTF-8.
|
||||
f.NonUTF8 = false
|
||||
default:
|
||||
// Might be UTF-8, might be some other encoding; preserve existing flag.
|
||||
// Some ZIP writers use UTF-8 encoding without setting the UTF-8 flag.
|
||||
// Since it is impossible to always distinguish valid UTF-8 from some
|
||||
// other encoding (e.g., GBK or Shift-JIS), we trust the flag.
|
||||
f.NonUTF8 = f.Flags&0x800 == 0
|
||||
}
|
||||
|
||||
needUSize := f.UncompressedSize == ^uint32(0)
|
||||
needCSize := f.CompressedSize == ^uint32(0)
|
||||
needHeaderOffset := f.headerOffset == int64(^uint32(0))
|
||||
|
||||
if len(f.Extra) > 0 {
|
||||
// Best effort to find what we need.
|
||||
// Other zip authors might not even follow the basic format,
|
||||
// and we'll just ignore the Extra content in that case.
|
||||
b := readBuf(f.Extra)
|
||||
for len(b) >= 4 { // need at least tag and size
|
||||
tag := b.uint16()
|
||||
size := b.uint16()
|
||||
if int(size) > len(b) {
|
||||
break
|
||||
}
|
||||
if tag == zip64ExtraId {
|
||||
// update directory values from the zip64 extra block.
|
||||
// They should only be consulted if the sizes read earlier
|
||||
// are maxed out.
|
||||
// See golang.org/issue/13367.
|
||||
eb := readBuf(b[:size])
|
||||
// Best effort to find what we need.
|
||||
// Other zip authors might not even follow the basic format,
|
||||
// and we'll just ignore the Extra content in that case.
|
||||
var modified time.Time
|
||||
parseExtras:
|
||||
for extra := readBuf(f.Extra); len(extra) >= 4; { // need at least tag and size
|
||||
fieldTag := extra.uint16()
|
||||
fieldSize := int(extra.uint16())
|
||||
if len(extra) < fieldSize {
|
||||
break
|
||||
}
|
||||
fieldBuf := extra.sub(fieldSize)
|
||||
|
||||
if needUSize {
|
||||
needUSize = false
|
||||
if len(eb) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.UncompressedSize64 = eb.uint64()
|
||||
switch fieldTag {
|
||||
case zip64ExtraID:
|
||||
// update directory values from the zip64 extra block.
|
||||
// They should only be consulted if the sizes read earlier
|
||||
// are maxed out.
|
||||
// See golang.org/issue/13367.
|
||||
if needUSize {
|
||||
needUSize = false
|
||||
if len(fieldBuf) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
if needCSize {
|
||||
needCSize = false
|
||||
if len(eb) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.CompressedSize64 = eb.uint64()
|
||||
}
|
||||
if needHeaderOffset {
|
||||
needHeaderOffset = false
|
||||
if len(eb) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.headerOffset = int64(eb.uint64())
|
||||
}
|
||||
break
|
||||
f.UncompressedSize64 = fieldBuf.uint64()
|
||||
}
|
||||
b = b[size:]
|
||||
if needCSize {
|
||||
needCSize = false
|
||||
if len(fieldBuf) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.CompressedSize64 = fieldBuf.uint64()
|
||||
}
|
||||
if needHeaderOffset {
|
||||
needHeaderOffset = false
|
||||
if len(fieldBuf) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.headerOffset = int64(fieldBuf.uint64())
|
||||
}
|
||||
case ntfsExtraID:
|
||||
if len(fieldBuf) < 4 {
|
||||
continue parseExtras
|
||||
}
|
||||
fieldBuf.uint32() // reserved (ignored)
|
||||
for len(fieldBuf) >= 4 { // need at least tag and size
|
||||
attrTag := fieldBuf.uint16()
|
||||
attrSize := int(fieldBuf.uint16())
|
||||
if len(fieldBuf) < attrSize {
|
||||
continue parseExtras
|
||||
}
|
||||
attrBuf := fieldBuf.sub(attrSize)
|
||||
if attrTag != 1 || attrSize != 24 {
|
||||
continue // Ignore irrelevant attributes
|
||||
}
|
||||
|
||||
const ticksPerSecond = 1e7 // Windows timestamp resolution
|
||||
ts := int64(attrBuf.uint64()) // ModTime since Windows epoch
|
||||
secs := int64(ts / ticksPerSecond)
|
||||
nsecs := (1e9 / ticksPerSecond) * int64(ts%ticksPerSecond)
|
||||
epoch := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
modified = time.Unix(epoch.Unix()+secs, nsecs)
|
||||
}
|
||||
case unixExtraID:
|
||||
if len(fieldBuf) < 8 {
|
||||
continue parseExtras
|
||||
}
|
||||
fieldBuf.uint32() // AcTime (ignored)
|
||||
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
|
||||
modified = time.Unix(ts, 0)
|
||||
case extTimeExtraID:
|
||||
if len(fieldBuf) < 5 || fieldBuf.uint8()&1 == 0 {
|
||||
continue parseExtras
|
||||
}
|
||||
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
|
||||
modified = time.Unix(ts, 0)
|
||||
case infoZipUnixExtraID:
|
||||
if len(fieldBuf) < 4 {
|
||||
continue parseExtras
|
||||
}
|
||||
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
|
||||
modified = time.Unix(ts, 0)
|
||||
}
|
||||
}
|
||||
|
||||
msdosModified := msDosTimeToTime(f.ModifiedDate, f.ModifiedTime)
|
||||
f.Modified = msdosModified
|
||||
if !modified.IsZero() {
|
||||
f.Modified = modified.UTC()
|
||||
|
||||
// If legacy MS-DOS timestamps are set, we can use the delta between
|
||||
// the legacy and extended versions to estimate timezone offset.
|
||||
//
|
||||
// A non-UTC timezone is always used (even if offset is zero).
|
||||
// Thus, FileHeader.Modified.Location() == time.UTC is useful for
|
||||
// determining whether extended timestamps are present.
|
||||
// This is necessary for users that need to do additional time
|
||||
// calculations when dealing with legacy ZIP formats.
|
||||
if f.ModifiedTime != 0 || f.ModifiedDate != 0 {
|
||||
f.Modified = modified.In(timeZone(msdosModified.Sub(modified)))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -508,6 +585,12 @@ func findSignatureInBlock(b []byte) int {
|
|||
|
||||
type readBuf []byte
|
||||
|
||||
func (b *readBuf) uint8() uint8 {
|
||||
v := (*b)[0]
|
||||
*b = (*b)[1:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint16() uint16 {
|
||||
v := binary.LittleEndian.Uint16(*b)
|
||||
*b = (*b)[2:]
|
||||
|
@ -525,3 +608,9 @@ func (b *readBuf) uint64() uint64 {
|
|||
*b = (*b)[8:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) sub(n int) readBuf {
|
||||
b2 := (*b)[:n]
|
||||
*b = (*b)[n:]
|
||||
return b2
|
||||
}
|
||||
|
|
|
@ -27,9 +27,11 @@ type ZipTest struct {
|
|||
}
|
||||
|
||||
type ZipTestFile struct {
|
||||
Name string
|
||||
Mode os.FileMode
|
||||
Mtime string // optional, modified time in format "mm-dd-yy hh:mm:ss"
|
||||
Name string
|
||||
Mode os.FileMode
|
||||
NonUTF8 bool
|
||||
ModTime time.Time
|
||||
Modified time.Time
|
||||
|
||||
// Information describing expected zip file content.
|
||||
// First, reading the entire content should produce the error ContentErr.
|
||||
|
@ -47,32 +49,22 @@ type ZipTestFile struct {
|
|||
Size uint64
|
||||
}
|
||||
|
||||
// Caution: The Mtime values found for the test files should correspond to
|
||||
// the values listed with unzip -l <zipfile>. However, the values
|
||||
// listed by unzip appear to be off by some hours. When creating
|
||||
// fresh test files and testing them, this issue is not present.
|
||||
// The test files were created in Sydney, so there might be a time
|
||||
// zone issue. The time zone information does have to be encoded
|
||||
// somewhere, because otherwise unzip -l could not provide a different
|
||||
// time from what the archive/zip package provides, but there appears
|
||||
// to be no documentation about this.
|
||||
|
||||
var tests = []ZipTest{
|
||||
{
|
||||
Name: "test.zip",
|
||||
Comment: "This is a zipfile comment.",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "test.txt",
|
||||
Content: []byte("This is a test text file.\n"),
|
||||
Mtime: "09-05-10 12:12:02",
|
||||
Mode: 0644,
|
||||
Name: "test.txt",
|
||||
Content: []byte("This is a test text file.\n"),
|
||||
Modified: time.Date(2010, 9, 5, 12, 12, 1, 0, timeZone(+10*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
{
|
||||
Name: "gophercolor16x16.png",
|
||||
File: "gophercolor16x16.png",
|
||||
Mtime: "09-05-10 15:52:58",
|
||||
Mode: 0644,
|
||||
Name: "gophercolor16x16.png",
|
||||
File: "gophercolor16x16.png",
|
||||
Modified: time.Date(2010, 9, 5, 15, 52, 58, 0, timeZone(+10*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -81,16 +73,16 @@ var tests = []ZipTest{
|
|||
Comment: "This is a zipfile comment.",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "test.txt",
|
||||
Content: []byte("This is a test text file.\n"),
|
||||
Mtime: "09-05-10 12:12:02",
|
||||
Mode: 0644,
|
||||
Name: "test.txt",
|
||||
Content: []byte("This is a test text file.\n"),
|
||||
Modified: time.Date(2010, 9, 5, 12, 12, 1, 0, timeZone(+10*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
{
|
||||
Name: "gophercolor16x16.png",
|
||||
File: "gophercolor16x16.png",
|
||||
Mtime: "09-05-10 15:52:58",
|
||||
Mode: 0644,
|
||||
Name: "gophercolor16x16.png",
|
||||
File: "gophercolor16x16.png",
|
||||
Modified: time.Date(2010, 9, 5, 15, 52, 58, 0, timeZone(+10*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -99,10 +91,10 @@ var tests = []ZipTest{
|
|||
Source: returnRecursiveZip,
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "r/r.zip",
|
||||
Content: rZipBytes(),
|
||||
Mtime: "03-04-10 00:24:16",
|
||||
Mode: 0666,
|
||||
Name: "r/r.zip",
|
||||
Content: rZipBytes(),
|
||||
Modified: time.Date(2010, 3, 4, 0, 24, 16, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -110,9 +102,10 @@ var tests = []ZipTest{
|
|||
Name: "symlink.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "symlink",
|
||||
Content: []byte("../target"),
|
||||
Mode: 0777 | os.ModeSymlink,
|
||||
Name: "symlink",
|
||||
Content: []byte("../target"),
|
||||
Modified: time.Date(2012, 2, 3, 19, 56, 48, 0, timeZone(-2*time.Hour)),
|
||||
Mode: 0777 | os.ModeSymlink,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -127,22 +120,72 @@ var tests = []ZipTest{
|
|||
Name: "dd.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "filename",
|
||||
Content: []byte("This is a test textfile.\n"),
|
||||
Mtime: "02-02-11 13:06:20",
|
||||
Mode: 0666,
|
||||
Name: "filename",
|
||||
Content: []byte("This is a test textfile.\n"),
|
||||
Modified: time.Date(2011, 2, 2, 13, 6, 20, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// created in windows XP file manager.
|
||||
Name: "winxp.zip",
|
||||
File: crossPlatform,
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "hello",
|
||||
Content: []byte("world \r\n"),
|
||||
Modified: time.Date(2011, 12, 8, 10, 4, 24, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
},
|
||||
{
|
||||
Name: "dir/bar",
|
||||
Content: []byte("foo \r\n"),
|
||||
Modified: time.Date(2011, 12, 8, 10, 4, 50, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
},
|
||||
{
|
||||
Name: "dir/empty/",
|
||||
Content: []byte{},
|
||||
Modified: time.Date(2011, 12, 8, 10, 8, 6, 0, time.UTC),
|
||||
Mode: os.ModeDir | 0777,
|
||||
},
|
||||
{
|
||||
Name: "readonly",
|
||||
Content: []byte("important \r\n"),
|
||||
Modified: time.Date(2011, 12, 8, 10, 6, 8, 0, time.UTC),
|
||||
Mode: 0444,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// created by Zip 3.0 under Linux
|
||||
Name: "unix.zip",
|
||||
File: crossPlatform,
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "hello",
|
||||
Content: []byte("world \r\n"),
|
||||
Modified: time.Date(2011, 12, 8, 10, 4, 24, 0, timeZone(0)),
|
||||
Mode: 0666,
|
||||
},
|
||||
{
|
||||
Name: "dir/bar",
|
||||
Content: []byte("foo \r\n"),
|
||||
Modified: time.Date(2011, 12, 8, 10, 4, 50, 0, timeZone(0)),
|
||||
Mode: 0666,
|
||||
},
|
||||
{
|
||||
Name: "dir/empty/",
|
||||
Content: []byte{},
|
||||
Modified: time.Date(2011, 12, 8, 10, 8, 6, 0, timeZone(0)),
|
||||
Mode: os.ModeDir | 0777,
|
||||
},
|
||||
{
|
||||
Name: "readonly",
|
||||
Content: []byte("important \r\n"),
|
||||
Modified: time.Date(2011, 12, 8, 10, 6, 8, 0, timeZone(0)),
|
||||
Mode: 0444,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// created by Go, before we wrote the "optional" data
|
||||
|
@ -150,16 +193,16 @@ var tests = []ZipTest{
|
|||
Name: "go-no-datadesc-sig.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "foo.txt",
|
||||
Content: []byte("foo\n"),
|
||||
Mtime: "03-08-12 16:59:10",
|
||||
Mode: 0644,
|
||||
Name: "foo.txt",
|
||||
Content: []byte("foo\n"),
|
||||
Modified: time.Date(2012, 3, 8, 16, 59, 10, 0, timeZone(-8*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
{
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Mtime: "03-08-12 16:59:12",
|
||||
Mode: 0644,
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Modified: time.Date(2012, 3, 8, 16, 59, 12, 0, timeZone(-8*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -169,14 +212,16 @@ var tests = []ZipTest{
|
|||
Name: "go-with-datadesc-sig.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "foo.txt",
|
||||
Content: []byte("foo\n"),
|
||||
Mode: 0666,
|
||||
Name: "foo.txt",
|
||||
Content: []byte("foo\n"),
|
||||
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
},
|
||||
{
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Mode: 0666,
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -187,13 +232,15 @@ var tests = []ZipTest{
|
|||
{
|
||||
Name: "foo.txt",
|
||||
Content: []byte("foo\n"),
|
||||
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
ContentErr: ErrChecksum,
|
||||
},
|
||||
{
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Mode: 0666,
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -203,16 +250,16 @@ var tests = []ZipTest{
|
|||
Name: "crc32-not-streamed.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "foo.txt",
|
||||
Content: []byte("foo\n"),
|
||||
Mtime: "03-08-12 16:59:10",
|
||||
Mode: 0644,
|
||||
Name: "foo.txt",
|
||||
Content: []byte("foo\n"),
|
||||
Modified: time.Date(2012, 3, 8, 16, 59, 10, 0, timeZone(-8*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
{
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Mtime: "03-08-12 16:59:12",
|
||||
Mode: 0644,
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Modified: time.Date(2012, 3, 8, 16, 59, 12, 0, timeZone(-8*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -225,15 +272,15 @@ var tests = []ZipTest{
|
|||
{
|
||||
Name: "foo.txt",
|
||||
Content: []byte("foo\n"),
|
||||
Mtime: "03-08-12 16:59:10",
|
||||
Modified: time.Date(2012, 3, 8, 16, 59, 10, 0, timeZone(-8*time.Hour)),
|
||||
Mode: 0644,
|
||||
ContentErr: ErrChecksum,
|
||||
},
|
||||
{
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Mtime: "03-08-12 16:59:12",
|
||||
Mode: 0644,
|
||||
Name: "bar.txt",
|
||||
Content: []byte("bar\n"),
|
||||
Modified: time.Date(2012, 3, 8, 16, 59, 12, 0, timeZone(-8*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -241,10 +288,10 @@ var tests = []ZipTest{
|
|||
Name: "zip64.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "README",
|
||||
Content: []byte("This small file is in ZIP64 format.\n"),
|
||||
Mtime: "08-10-12 14:33:32",
|
||||
Mode: 0644,
|
||||
Name: "README",
|
||||
Content: []byte("This small file is in ZIP64 format.\n"),
|
||||
Modified: time.Date(2012, 8, 10, 14, 33, 32, 0, time.UTC),
|
||||
Mode: 0644,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -253,10 +300,10 @@ var tests = []ZipTest{
|
|||
Name: "zip64-2.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "README",
|
||||
Content: []byte("This small file is in ZIP64 format.\n"),
|
||||
Mtime: "08-10-12 14:33:32",
|
||||
Mode: 0644,
|
||||
Name: "README",
|
||||
Content: []byte("This small file is in ZIP64 format.\n"),
|
||||
Modified: time.Date(2012, 8, 10, 14, 33, 32, 0, timeZone(-4*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -266,41 +313,179 @@ var tests = []ZipTest{
|
|||
Source: returnBigZipBytes,
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "big.file",
|
||||
Content: nil,
|
||||
Size: 1<<32 - 1,
|
||||
Mode: 0666,
|
||||
Name: "big.file",
|
||||
Content: nil,
|
||||
Size: 1<<32 - 1,
|
||||
Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "utf8-7zip.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "世界",
|
||||
Content: []byte{},
|
||||
Mode: 0666,
|
||||
Modified: time.Date(2017, 11, 6, 13, 9, 27, 867862500, timeZone(-8*time.Hour)),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "utf8-infozip.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "世界",
|
||||
Content: []byte{},
|
||||
Mode: 0644,
|
||||
// Name is valid UTF-8, but format does not have UTF-8 flag set.
|
||||
// We don't do UTF-8 detection for multi-byte runes due to
|
||||
// false-positives with other encodings (e.g., Shift-JIS).
|
||||
// Format says encoding is not UTF-8, so we trust it.
|
||||
NonUTF8: true,
|
||||
Modified: time.Date(2017, 11, 6, 13, 9, 27, 0, timeZone(-8*time.Hour)),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "utf8-osx.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "世界",
|
||||
Content: []byte{},
|
||||
Mode: 0644,
|
||||
// Name is valid UTF-8, but format does not have UTF-8 set.
|
||||
NonUTF8: true,
|
||||
Modified: time.Date(2017, 11, 6, 13, 9, 27, 0, timeZone(-8*time.Hour)),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "utf8-winrar.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "世界",
|
||||
Content: []byte{},
|
||||
Mode: 0666,
|
||||
Modified: time.Date(2017, 11, 6, 13, 9, 27, 867862500, timeZone(-8*time.Hour)),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "utf8-winzip.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "世界",
|
||||
Content: []byte{},
|
||||
Mode: 0666,
|
||||
Modified: time.Date(2017, 11, 6, 13, 9, 27, 867000000, timeZone(-8*time.Hour)),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "time-7zip.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "test.txt",
|
||||
Content: []byte{},
|
||||
Size: 1<<32 - 1,
|
||||
Modified: time.Date(2017, 10, 31, 21, 11, 57, 244817900, timeZone(-7*time.Hour)),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "time-infozip.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "test.txt",
|
||||
Content: []byte{},
|
||||
Size: 1<<32 - 1,
|
||||
Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "time-osx.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "test.txt",
|
||||
Content: []byte{},
|
||||
Size: 1<<32 - 1,
|
||||
Modified: time.Date(2017, 10, 31, 21, 17, 27, 0, timeZone(-7*time.Hour)),
|
||||
Mode: 0644,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "time-win7.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "test.txt",
|
||||
Content: []byte{},
|
||||
Size: 1<<32 - 1,
|
||||
Modified: time.Date(2017, 10, 31, 21, 11, 58, 0, time.UTC),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "time-winrar.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "test.txt",
|
||||
Content: []byte{},
|
||||
Size: 1<<32 - 1,
|
||||
Modified: time.Date(2017, 10, 31, 21, 11, 57, 244817900, timeZone(-7*time.Hour)),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "time-winzip.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "test.txt",
|
||||
Content: []byte{},
|
||||
Size: 1<<32 - 1,
|
||||
Modified: time.Date(2017, 10, 31, 21, 11, 57, 244000000, timeZone(-7*time.Hour)),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "time-go.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "test.txt",
|
||||
Content: []byte{},
|
||||
Size: 1<<32 - 1,
|
||||
Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)),
|
||||
Mode: 0666,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "time-22738.zip",
|
||||
File: []ZipTestFile{
|
||||
{
|
||||
Name: "file",
|
||||
Content: []byte{},
|
||||
Mode: 0666,
|
||||
Modified: time.Date(1999, 12, 31, 19, 0, 0, 0, timeZone(-5*time.Hour)),
|
||||
ModTime: time.Date(1999, 12, 31, 19, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var crossPlatform = []ZipTestFile{
|
||||
{
|
||||
Name: "hello",
|
||||
Content: []byte("world \r\n"),
|
||||
Mode: 0666,
|
||||
},
|
||||
{
|
||||
Name: "dir/bar",
|
||||
Content: []byte("foo \r\n"),
|
||||
Mode: 0666,
|
||||
},
|
||||
{
|
||||
Name: "dir/empty/",
|
||||
Content: []byte{},
|
||||
Mode: os.ModeDir | 0777,
|
||||
},
|
||||
{
|
||||
Name: "readonly",
|
||||
Content: []byte("important \r\n"),
|
||||
Mode: 0444,
|
||||
},
|
||||
}
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
for _, zt := range tests {
|
||||
readTestZip(t, zt)
|
||||
t.Run(zt.Name, func(t *testing.T) {
|
||||
readTestZip(t, zt)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -319,7 +504,7 @@ func readTestZip(t *testing.T, zt ZipTest) {
|
|||
}
|
||||
}
|
||||
if err != zt.Error {
|
||||
t.Errorf("%s: error=%v, want %v", zt.Name, err, zt.Error)
|
||||
t.Errorf("error=%v, want %v", err, zt.Error)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -335,16 +520,19 @@ func readTestZip(t *testing.T, zt ZipTest) {
|
|||
}
|
||||
|
||||
if z.Comment != zt.Comment {
|
||||
t.Errorf("%s: comment=%q, want %q", zt.Name, z.Comment, zt.Comment)
|
||||
t.Errorf("comment=%q, want %q", z.Comment, zt.Comment)
|
||||
}
|
||||
if len(z.File) != len(zt.File) {
|
||||
t.Fatalf("%s: file count=%d, want %d", zt.Name, len(z.File), len(zt.File))
|
||||
t.Fatalf("file count=%d, want %d", len(z.File), len(zt.File))
|
||||
}
|
||||
|
||||
// test read of each file
|
||||
for i, ft := range zt.File {
|
||||
readTestFile(t, zt, ft, z.File[i])
|
||||
}
|
||||
if t.Failed() {
|
||||
return
|
||||
}
|
||||
|
||||
// test simultaneous reads
|
||||
n := 0
|
||||
|
@ -363,23 +551,24 @@ func readTestZip(t *testing.T, zt ZipTest) {
|
|||
}
|
||||
}
|
||||
|
||||
func equalTimeAndZone(t1, t2 time.Time) bool {
|
||||
name1, offset1 := t1.Zone()
|
||||
name2, offset2 := t2.Zone()
|
||||
return t1.Equal(t2) && name1 == name2 && offset1 == offset2
|
||||
}
|
||||
|
||||
func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
|
||||
if f.Name != ft.Name {
|
||||
t.Errorf("%s: name=%q, want %q", zt.Name, f.Name, ft.Name)
|
||||
t.Errorf("name=%q, want %q", f.Name, ft.Name)
|
||||
}
|
||||
if !ft.Modified.IsZero() && !equalTimeAndZone(f.Modified, ft.Modified) {
|
||||
t.Errorf("%s: Modified=%s, want %s", f.Name, f.Modified, ft.Modified)
|
||||
}
|
||||
if !ft.ModTime.IsZero() && !equalTimeAndZone(f.ModTime(), ft.ModTime) {
|
||||
t.Errorf("%s: ModTime=%s, want %s", f.Name, f.ModTime(), ft.ModTime)
|
||||
}
|
||||
|
||||
if ft.Mtime != "" {
|
||||
mtime, err := time.Parse("01-02-06 15:04:05", ft.Mtime)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if ft := f.ModTime(); !ft.Equal(mtime) {
|
||||
t.Errorf("%s: %s: mtime=%s, want %s", zt.Name, f.Name, ft, mtime)
|
||||
}
|
||||
}
|
||||
|
||||
testFileMode(t, zt.Name, f, ft.Mode)
|
||||
testFileMode(t, f, ft.Mode)
|
||||
|
||||
size := uint64(f.UncompressedSize)
|
||||
if size == uint32max {
|
||||
|
@ -390,7 +579,7 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
|
|||
|
||||
r, err := f.Open()
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", zt.Name, err)
|
||||
t.Errorf("%v", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -408,7 +597,7 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
|
|||
var b bytes.Buffer
|
||||
_, err = io.Copy(&b, r)
|
||||
if err != ft.ContentErr {
|
||||
t.Errorf("%s: copying contents: %v (want %v)", zt.Name, err, ft.ContentErr)
|
||||
t.Errorf("copying contents: %v (want %v)", err, ft.ContentErr)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
|
@ -440,12 +629,12 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
|
|||
}
|
||||
}
|
||||
|
||||
func testFileMode(t *testing.T, zipName string, f *File, want os.FileMode) {
|
||||
func testFileMode(t *testing.T, f *File, want os.FileMode) {
|
||||
mode := f.Mode()
|
||||
if want == 0 {
|
||||
t.Errorf("%s: %s mode: got %v, want none", zipName, f.Name, mode)
|
||||
t.Errorf("%s mode: got %v, want none", f.Name, mode)
|
||||
} else if mode != want {
|
||||
t.Errorf("%s: %s mode: want %v, got %v", zipName, f.Name, want, mode)
|
||||
t.Errorf("%s mode: want %v, got %v", f.Name, want, mode)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -27,8 +27,8 @@ import (
|
|||
|
||||
// Compression methods.
|
||||
const (
|
||||
Store uint16 = 0
|
||||
Deflate uint16 = 8
|
||||
Store uint16 = 0 // no compression
|
||||
Deflate uint16 = 8 // DEFLATE compressed
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -46,40 +46,79 @@ const (
|
|||
directory64LocLen = 20 //
|
||||
directory64EndLen = 56 // + extra
|
||||
|
||||
// Constants for the first byte in CreatorVersion
|
||||
// Constants for the first byte in CreatorVersion.
|
||||
creatorFAT = 0
|
||||
creatorUnix = 3
|
||||
creatorNTFS = 11
|
||||
creatorVFAT = 14
|
||||
creatorMacOSX = 19
|
||||
|
||||
// version numbers
|
||||
// Version numbers.
|
||||
zipVersion20 = 20 // 2.0
|
||||
zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)
|
||||
|
||||
// limits for non zip64 files
|
||||
// Limits for non zip64 files.
|
||||
uint16max = (1 << 16) - 1
|
||||
uint32max = (1 << 32) - 1
|
||||
|
||||
// extra header id's
|
||||
zip64ExtraId = 0x0001 // zip64 Extended Information Extra Field
|
||||
// Extra header IDs.
|
||||
//
|
||||
// IDs 0..31 are reserved for official use by PKWARE.
|
||||
// IDs above that range are defined by third-party vendors.
|
||||
// Since ZIP lacked high precision timestamps (nor a official specification
|
||||
// of the timezone used for the date fields), many competing extra fields
|
||||
// have been invented. Pervasive use effectively makes them "official".
|
||||
//
|
||||
// See http://mdfs.net/Docs/Comp/Archiving/Zip/ExtraField
|
||||
zip64ExtraID = 0x0001 // Zip64 extended information
|
||||
ntfsExtraID = 0x000a // NTFS
|
||||
unixExtraID = 0x000d // UNIX
|
||||
extTimeExtraID = 0x5455 // Extended timestamp
|
||||
infoZipUnixExtraID = 0x5855 // Info-ZIP Unix extension
|
||||
)
|
||||
|
||||
// FileHeader describes a file within a zip file.
|
||||
// See the zip spec for details.
|
||||
type FileHeader struct {
|
||||
// Name is the name of the file.
|
||||
// It must be a relative path: it must not start with a drive
|
||||
// letter (e.g. C:) or leading slash, and only forward slashes
|
||||
// are allowed.
|
||||
// It must be a relative path, not start with a drive letter (e.g. C:),
|
||||
// and must use forward slashes instead of back slashes.
|
||||
Name string
|
||||
|
||||
CreatorVersion uint16
|
||||
ReaderVersion uint16
|
||||
Flags uint16
|
||||
Method uint16
|
||||
ModifiedTime uint16 // MS-DOS time
|
||||
ModifiedDate uint16 // MS-DOS date
|
||||
// Comment is any arbitrary user-defined string shorter than 64KiB.
|
||||
Comment string
|
||||
|
||||
// NonUTF8 indicates that Name and Comment are not encoded in UTF-8.
|
||||
//
|
||||
// By specification, the only other encoding permitted should be CP-437,
|
||||
// but historically many ZIP readers interpret Name and Comment as whatever
|
||||
// the system's local character encoding happens to be.
|
||||
//
|
||||
// This flag should only be set if the user intends to encode a non-portable
|
||||
// ZIP file for a specific localized region. Otherwise, the Writer
|
||||
// automatically sets the ZIP format's UTF-8 flag for valid UTF-8 strings.
|
||||
NonUTF8 bool
|
||||
|
||||
CreatorVersion uint16
|
||||
ReaderVersion uint16
|
||||
Flags uint16
|
||||
|
||||
// Method is the compression method. If zero, Store is used.
|
||||
Method uint16
|
||||
|
||||
// Modified is the modified time of the file.
|
||||
//
|
||||
// When reading, an extended timestamp is preferred over the legacy MS-DOS
|
||||
// date field, and the offset between the times is used as the timezone.
|
||||
// If only the MS-DOS date is present, the timezone is assumed to be UTC.
|
||||
//
|
||||
// When writing, an extended timestamp (which is timezone-agnostic) is
|
||||
// always emitted. The legacy MS-DOS date field is encoded according to the
|
||||
// location of the Modified time.
|
||||
Modified time.Time
|
||||
ModifiedTime uint16 // Deprecated: Legacy MS-DOS date; use Modified instead.
|
||||
ModifiedDate uint16 // Deprecated: Legacy MS-DOS time; use Modified instead.
|
||||
|
||||
CRC32 uint32
|
||||
CompressedSize uint32 // Deprecated: Use CompressedSize64 instead.
|
||||
UncompressedSize uint32 // Deprecated: Use UncompressedSize64 instead.
|
||||
|
@ -87,7 +126,6 @@ type FileHeader struct {
|
|||
UncompressedSize64 uint64
|
||||
Extra []byte
|
||||
ExternalAttrs uint32 // Meaning depends on CreatorVersion
|
||||
Comment string
|
||||
}
|
||||
|
||||
// FileInfo returns an os.FileInfo for the FileHeader.
|
||||
|
@ -117,6 +155,8 @@ func (fi headerFileInfo) Sys() interface{} { return fi.fh }
|
|||
// Because os.FileInfo's Name method returns only the base name of
|
||||
// the file it describes, it may be necessary to modify the Name field
|
||||
// of the returned header to provide the full path name of the file.
|
||||
// If compression is desired, callers should set the FileHeader.Method
|
||||
// field; it is unset by default.
|
||||
func FileInfoHeader(fi os.FileInfo) (*FileHeader, error) {
|
||||
size := fi.Size()
|
||||
fh := &FileHeader{
|
||||
|
@ -144,6 +184,21 @@ type directoryEnd struct {
|
|||
comment string
|
||||
}
|
||||
|
||||
// timeZone returns a *time.Location based on the provided offset.
|
||||
// If the offset is non-sensible, then this uses an offset of zero.
|
||||
func timeZone(offset time.Duration) *time.Location {
|
||||
const (
|
||||
minOffset = -12 * time.Hour // E.g., Baker island at -12:00
|
||||
maxOffset = +14 * time.Hour // E.g., Line island at +14:00
|
||||
offsetAlias = 15 * time.Minute // E.g., Nepal at +5:45
|
||||
)
|
||||
offset = offset.Round(offsetAlias)
|
||||
if offset < minOffset || maxOffset < offset {
|
||||
offset = 0
|
||||
}
|
||||
return time.FixedZone("", int(offset/time.Second))
|
||||
}
|
||||
|
||||
// msDosTimeToTime converts an MS-DOS date and time into a time.Time.
|
||||
// The resolution is 2s.
|
||||
// See: http://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx
|
||||
|
@ -168,21 +223,26 @@ func msDosTimeToTime(dosDate, dosTime uint16) time.Time {
|
|||
// The resolution is 2s.
|
||||
// See: http://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx
|
||||
func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) {
|
||||
t = t.In(time.UTC)
|
||||
fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
|
||||
fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
|
||||
return
|
||||
}
|
||||
|
||||
// ModTime returns the modification time in UTC.
|
||||
// The resolution is 2s.
|
||||
// ModTime returns the modification time in UTC using the legacy
|
||||
// ModifiedDate and ModifiedTime fields.
|
||||
//
|
||||
// Deprecated: Use Modified instead.
|
||||
func (h *FileHeader) ModTime() time.Time {
|
||||
return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime)
|
||||
}
|
||||
|
||||
// SetModTime sets the ModifiedTime and ModifiedDate fields to the given time in UTC.
|
||||
// The resolution is 2s.
|
||||
// SetModTime sets the Modified, ModifiedTime, and ModifiedDate fields
|
||||
// to the given time in UTC.
|
||||
//
|
||||
// Deprecated: Use Modified instead.
|
||||
func (h *FileHeader) SetModTime(t time.Time) {
|
||||
t = t.UTC() // Convert to UTC for compatibility
|
||||
h.Modified = t
|
||||
h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t)
|
||||
}
|
||||
|
||||
|
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -14,6 +14,11 @@ import (
|
|||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var (
|
||||
errLongName = errors.New("zip: FileHeader.Name too long")
|
||||
errLongExtra = errors.New("zip: FileHeader.Extra too long")
|
||||
)
|
||||
|
||||
// Writer implements a zip file writer.
|
||||
type Writer struct {
|
||||
cw *countWriter
|
||||
|
@ -21,6 +26,7 @@ type Writer struct {
|
|||
last *fileWriter
|
||||
closed bool
|
||||
compressors map[uint16]Compressor
|
||||
comment string
|
||||
|
||||
// testHookCloseSizeOffset if non-nil is called with the size
|
||||
// of offset of the central directory at Close.
|
||||
|
@ -54,6 +60,16 @@ func (w *Writer) Flush() error {
|
|||
return w.cw.w.(*bufio.Writer).Flush()
|
||||
}
|
||||
|
||||
// SetComment sets the end-of-central-directory comment field.
|
||||
// It can only be called before Close.
|
||||
func (w *Writer) SetComment(comment string) error {
|
||||
if len(comment) > uint16max {
|
||||
return errors.New("zip: Writer.Comment too long")
|
||||
}
|
||||
w.comment = comment
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close finishes writing the zip file by writing the central directory.
|
||||
// It does not (and cannot) close the underlying writer.
|
||||
func (w *Writer) Close() error {
|
||||
|
@ -91,7 +107,7 @@ func (w *Writer) Close() error {
|
|||
// append a zip64 extra block to Extra
|
||||
var buf [28]byte // 2x uint16 + 3x uint64
|
||||
eb := writeBuf(buf[:])
|
||||
eb.uint16(zip64ExtraId)
|
||||
eb.uint16(zip64ExtraID)
|
||||
eb.uint16(24) // size = 3x uint64
|
||||
eb.uint64(h.UncompressedSize64)
|
||||
eb.uint64(h.CompressedSize64)
|
||||
|
@ -172,21 +188,25 @@ func (w *Writer) Close() error {
|
|||
var buf [directoryEndLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
b.uint32(uint32(directoryEndSignature))
|
||||
b = b[4:] // skip over disk number and first disk number (2x uint16)
|
||||
b.uint16(uint16(records)) // number of entries this disk
|
||||
b.uint16(uint16(records)) // number of entries total
|
||||
b.uint32(uint32(size)) // size of directory
|
||||
b.uint32(uint32(offset)) // start of directory
|
||||
// skipped size of comment (always zero)
|
||||
b = b[4:] // skip over disk number and first disk number (2x uint16)
|
||||
b.uint16(uint16(records)) // number of entries this disk
|
||||
b.uint16(uint16(records)) // number of entries total
|
||||
b.uint32(uint32(size)) // size of directory
|
||||
b.uint32(uint32(offset)) // start of directory
|
||||
b.uint16(uint16(len(w.comment))) // byte size of EOCD comment
|
||||
if _, err := w.cw.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w.cw, w.comment); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return w.cw.w.(*bufio.Writer).Flush()
|
||||
}
|
||||
|
||||
// Create adds a file to the zip file using the provided name.
|
||||
// It returns a Writer to which the file contents should be written.
|
||||
// The file contents will be compressed using the Deflate method.
|
||||
// The name must be a relative path: it must not start with a drive
|
||||
// letter (e.g. C:) or leading slash, and only forward slashes are
|
||||
// allowed.
|
||||
|
@ -200,27 +220,36 @@ func (w *Writer) Create(name string) (io.Writer, error) {
|
|||
return w.CreateHeader(header)
|
||||
}
|
||||
|
||||
func hasValidUTF8(s string) bool {
|
||||
n := 0
|
||||
for _, r := range s {
|
||||
// By default, ZIP uses CP437, which is only identical to ASCII for the printable characters.
|
||||
if r < 0x20 || r >= 0x7f {
|
||||
if !utf8.ValidRune(r) {
|
||||
return false
|
||||
// detectUTF8 reports whether s is a valid UTF-8 string, and whether the string
|
||||
// must be considered UTF-8 encoding (i.e., not compatible with CP-437, ASCII,
|
||||
// or any other common encoding).
|
||||
func detectUTF8(s string) (valid, require bool) {
|
||||
for i := 0; i < len(s); {
|
||||
r, size := utf8.DecodeRuneInString(s[i:])
|
||||
i += size
|
||||
// Officially, ZIP uses CP-437, but many readers use the system's
|
||||
// local character encoding. Most encoding are compatible with a large
|
||||
// subset of CP-437, which itself is ASCII-like.
|
||||
//
|
||||
// Forbid 0x7e and 0x5c since EUC-KR and Shift-JIS replace those
|
||||
// characters with localized currency and overline characters.
|
||||
if r < 0x20 || r > 0x7d || r == 0x5c {
|
||||
if !utf8.ValidRune(r) || (r == utf8.RuneError && size == 1) {
|
||||
return false, false
|
||||
}
|
||||
n++
|
||||
require = true
|
||||
}
|
||||
}
|
||||
return n > 0
|
||||
return true, require
|
||||
}
|
||||
|
||||
// CreateHeader adds a file to the zip file using the provided FileHeader
|
||||
// for the file metadata.
|
||||
// It returns a Writer to which the file contents should be written.
|
||||
// CreateHeader adds a file to the zip archive using the provided FileHeader
|
||||
// for the file metadata. Writer takes ownership of fh and may mutate
|
||||
// its fields. The caller must not modify fh after calling CreateHeader.
|
||||
//
|
||||
// This returns a Writer to which the file contents should be written.
|
||||
// The file's contents must be written to the io.Writer before the next
|
||||
// call to Create, CreateHeader, or Close. The provided FileHeader fh
|
||||
// must not be modified after a call to CreateHeader.
|
||||
// call to Create, CreateHeader, or Close.
|
||||
func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
|
||||
if w.last != nil && !w.last.closed {
|
||||
if err := w.last.close(); err != nil {
|
||||
|
@ -234,13 +263,62 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
|
|||
|
||||
fh.Flags |= 0x8 // we will write a data descriptor
|
||||
|
||||
if hasValidUTF8(fh.Name) || hasValidUTF8(fh.Comment) {
|
||||
fh.Flags |= 0x800 // filename or comment have valid utf-8 string
|
||||
// The ZIP format has a sad state of affairs regarding character encoding.
|
||||
// Officially, the name and comment fields are supposed to be encoded
|
||||
// in CP-437 (which is mostly compatible with ASCII), unless the UTF-8
|
||||
// flag bit is set. However, there are several problems:
|
||||
//
|
||||
// * Many ZIP readers still do not support UTF-8.
|
||||
// * If the UTF-8 flag is cleared, several readers simply interpret the
|
||||
// name and comment fields as whatever the local system encoding is.
|
||||
//
|
||||
// In order to avoid breaking readers without UTF-8 support,
|
||||
// we avoid setting the UTF-8 flag if the strings are CP-437 compatible.
|
||||
// However, if the strings require multibyte UTF-8 encoding and is a
|
||||
// valid UTF-8 string, then we set the UTF-8 bit.
|
||||
//
|
||||
// For the case, where the user explicitly wants to specify the encoding
|
||||
// as UTF-8, they will need to set the flag bit themselves.
|
||||
utf8Valid1, utf8Require1 := detectUTF8(fh.Name)
|
||||
utf8Valid2, utf8Require2 := detectUTF8(fh.Comment)
|
||||
switch {
|
||||
case fh.NonUTF8:
|
||||
fh.Flags &^= 0x800
|
||||
case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2):
|
||||
fh.Flags |= 0x800
|
||||
}
|
||||
|
||||
fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
|
||||
fh.ReaderVersion = zipVersion20
|
||||
|
||||
// If Modified is set, this takes precedence over MS-DOS timestamp fields.
|
||||
if !fh.Modified.IsZero() {
|
||||
// Contrary to the FileHeader.SetModTime method, we intentionally
|
||||
// do not convert to UTC, because we assume the user intends to encode
|
||||
// the date using the specified timezone. A user may want this control
|
||||
// because many legacy ZIP readers interpret the timestamp according
|
||||
// to the local timezone.
|
||||
//
|
||||
// The timezone is only non-UTC if a user directly sets the Modified
|
||||
// field directly themselves. All other approaches sets UTC.
|
||||
fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified)
|
||||
|
||||
// Use "extended timestamp" format since this is what Info-ZIP uses.
|
||||
// Nearly every major ZIP implementation uses a different format,
|
||||
// but at least most seem to be able to understand the other formats.
|
||||
//
|
||||
// This format happens to be identical for both local and central header
|
||||
// if modification time is the only timestamp being encoded.
|
||||
var mbuf [9]byte // 2*SizeOf(uint16) + SizeOf(uint8) + SizeOf(uint32)
|
||||
mt := uint32(fh.Modified.Unix())
|
||||
eb := writeBuf(mbuf[:])
|
||||
eb.uint16(extTimeExtraID)
|
||||
eb.uint16(5) // Size: SizeOf(uint8) + SizeOf(uint32)
|
||||
eb.uint8(1) // Flags: ModTime
|
||||
eb.uint32(mt) // ModTime
|
||||
fh.Extra = append(fh.Extra, mbuf[:]...)
|
||||
}
|
||||
|
||||
fw := &fileWriter{
|
||||
zipw: w.cw,
|
||||
compCount: &countWriter{w: w.cw},
|
||||
|
@ -273,6 +351,14 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
|
|||
}
|
||||
|
||||
func writeHeader(w io.Writer, h *FileHeader) error {
|
||||
const maxUint16 = 1<<16 - 1
|
||||
if len(h.Name) > maxUint16 {
|
||||
return errLongName
|
||||
}
|
||||
if len(h.Extra) > maxUint16 {
|
||||
return errLongExtra
|
||||
}
|
||||
|
||||
var buf [fileHeaderLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
b.uint32(uint32(fileHeaderSignature))
|
||||
|
@ -402,6 +488,11 @@ func (w nopCloser) Close() error {
|
|||
|
||||
type writeBuf []byte
|
||||
|
||||
func (b *writeBuf) uint8(v uint8) {
|
||||
(*b)[0] = v
|
||||
*b = (*b)[1:]
|
||||
}
|
||||
|
||||
func (b *writeBuf) uint16(v uint16) {
|
||||
binary.LittleEndian.PutUint16(*b, v)
|
||||
*b = (*b)[2:]
|
||||
|
|
|
@ -6,11 +6,14 @@ package zip
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TODO(adg): a more sophisticated test suite
|
||||
|
@ -57,8 +60,8 @@ var writeTests = []WriteTest{
|
|||
|
||||
func TestWriter(t *testing.T) {
|
||||
largeData := make([]byte, 1<<17)
|
||||
for i := range largeData {
|
||||
largeData[i] = byte(rand.Int())
|
||||
if _, err := rand.Read(largeData); err != nil {
|
||||
t.Fatal("rand.Read failed:", err)
|
||||
}
|
||||
writeTests[1].Data = largeData
|
||||
defer func() {
|
||||
|
@ -87,31 +90,100 @@ func TestWriter(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestWriterComment is test for EOCD comment read/write.
|
||||
func TestWriterComment(t *testing.T) {
|
||||
var tests = []struct {
|
||||
comment string
|
||||
ok bool
|
||||
}{
|
||||
{"hi, hello", true},
|
||||
{"hi, こんにちわ", true},
|
||||
{strings.Repeat("a", uint16max), true},
|
||||
{strings.Repeat("a", uint16max+1), false},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
// write a zip file
|
||||
buf := new(bytes.Buffer)
|
||||
w := NewWriter(buf)
|
||||
if err := w.SetComment(test.comment); err != nil {
|
||||
if test.ok {
|
||||
t.Fatalf("SetComment: unexpected error %v", err)
|
||||
}
|
||||
continue
|
||||
} else {
|
||||
if !test.ok {
|
||||
t.Fatalf("SetComment: unexpected success, want error")
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.Close(); test.ok == (err != nil) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if w.closed != test.ok {
|
||||
t.Fatalf("Writer.closed: got %v, want %v", w.closed, test.ok)
|
||||
}
|
||||
|
||||
// skip read test in failure cases
|
||||
if !test.ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// read it back
|
||||
r, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Comment != test.comment {
|
||||
t.Fatalf("Reader.Comment: got %v, want %v", r.Comment, test.comment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterUTF8(t *testing.T) {
|
||||
var utf8Tests = []struct {
|
||||
name string
|
||||
comment string
|
||||
expect uint16
|
||||
nonUTF8 bool
|
||||
flags uint16
|
||||
}{
|
||||
{
|
||||
name: "hi, hello",
|
||||
comment: "in the world",
|
||||
expect: 0x8,
|
||||
flags: 0x8,
|
||||
},
|
||||
{
|
||||
name: "hi, こんにちわ",
|
||||
comment: "in the world",
|
||||
expect: 0x808,
|
||||
flags: 0x808,
|
||||
},
|
||||
{
|
||||
name: "hi, こんにちわ",
|
||||
comment: "in the world",
|
||||
nonUTF8: true,
|
||||
flags: 0x8,
|
||||
},
|
||||
{
|
||||
name: "hi, hello",
|
||||
comment: "in the 世界",
|
||||
expect: 0x808,
|
||||
flags: 0x808,
|
||||
},
|
||||
{
|
||||
name: "hi, こんにちわ",
|
||||
comment: "in the 世界",
|
||||
expect: 0x808,
|
||||
flags: 0x808,
|
||||
},
|
||||
{
|
||||
name: "the replacement rune is <20>",
|
||||
comment: "the replacement rune is <20>",
|
||||
flags: 0x808,
|
||||
},
|
||||
{
|
||||
// Name is Japanese encoded in Shift JIS.
|
||||
name: "\x93\xfa\x96{\x8c\xea.txt",
|
||||
comment: "in the 世界",
|
||||
flags: 0x008, // UTF-8 must not be set
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -123,6 +195,7 @@ func TestWriterUTF8(t *testing.T) {
|
|||
h := &FileHeader{
|
||||
Name: test.name,
|
||||
Comment: test.comment,
|
||||
NonUTF8: test.nonUTF8,
|
||||
Method: Deflate,
|
||||
}
|
||||
w, err := w.CreateHeader(h)
|
||||
|
@ -142,18 +215,41 @@ func TestWriterUTF8(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
for i, test := range utf8Tests {
|
||||
got := r.File[i].Flags
|
||||
t.Logf("name %v, comment %v", test.name, test.comment)
|
||||
if got != test.expect {
|
||||
t.Fatalf("Flags: got %v, want %v", got, test.expect)
|
||||
flags := r.File[i].Flags
|
||||
if flags != test.flags {
|
||||
t.Errorf("CreateHeader(name=%q comment=%q nonUTF8=%v): flags=%#x, want %#x", test.name, test.comment, test.nonUTF8, flags, test.flags)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterTime(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
h := &FileHeader{
|
||||
Name: "test.txt",
|
||||
Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)),
|
||||
}
|
||||
w := NewWriter(&buf)
|
||||
if _, err := w.CreateHeader(h); err != nil {
|
||||
t.Fatalf("unexpected CreateHeader error: %v", err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatalf("unexpected Close error: %v", err)
|
||||
}
|
||||
|
||||
want, err := ioutil.ReadFile("testdata/time-go.zip")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected ReadFile error: %v", err)
|
||||
}
|
||||
if got := buf.Bytes(); !bytes.Equal(got, want) {
|
||||
fmt.Printf("%x\n%x\n", got, want)
|
||||
t.Error("contents of time-go.zip differ")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterOffset(t *testing.T) {
|
||||
largeData := make([]byte, 1<<17)
|
||||
for i := range largeData {
|
||||
largeData[i] = byte(rand.Int())
|
||||
if _, err := rand.Read(largeData); err != nil {
|
||||
t.Fatal("rand.Read failed:", err)
|
||||
}
|
||||
writeTests[1].Data = largeData
|
||||
defer func() {
|
||||
|
@ -225,7 +321,7 @@ func testReadFile(t *testing.T, f *File, wt *WriteTest) {
|
|||
if f.Name != wt.Name {
|
||||
t.Fatalf("File name: got %q, want %q", f.Name, wt.Name)
|
||||
}
|
||||
testFileMode(t, wt.Name, f, wt.Mode)
|
||||
testFileMode(t, f, wt.Mode)
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
t.Fatal("opening:", err)
|
||||
|
|
|
@ -645,16 +645,54 @@ func TestHeaderTooShort(t *testing.T) {
|
|||
h := FileHeader{
|
||||
Name: "foo.txt",
|
||||
Method: Deflate,
|
||||
Extra: []byte{zip64ExtraId}, // missing size and second half of tag, but Extra is best-effort parsing
|
||||
Extra: []byte{zip64ExtraID}, // missing size and second half of tag, but Extra is best-effort parsing
|
||||
}
|
||||
testValidHeader(&h, t)
|
||||
}
|
||||
|
||||
func TestHeaderTooLongErr(t *testing.T) {
|
||||
var headerTests = []struct {
|
||||
name string
|
||||
extra []byte
|
||||
wanterr error
|
||||
}{
|
||||
{
|
||||
name: strings.Repeat("x", 1<<16),
|
||||
extra: []byte{},
|
||||
wanterr: errLongName,
|
||||
},
|
||||
{
|
||||
name: "long_extra",
|
||||
extra: bytes.Repeat([]byte{0xff}, 1<<16),
|
||||
wanterr: errLongExtra,
|
||||
},
|
||||
}
|
||||
|
||||
// write a zip file
|
||||
buf := new(bytes.Buffer)
|
||||
w := NewWriter(buf)
|
||||
|
||||
for _, test := range headerTests {
|
||||
h := &FileHeader{
|
||||
Name: test.name,
|
||||
Extra: test.extra,
|
||||
}
|
||||
_, err := w.CreateHeader(h)
|
||||
if err != test.wanterr {
|
||||
t.Errorf("error=%v, want %v", err, test.wanterr)
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderIgnoredSize(t *testing.T) {
|
||||
h := FileHeader{
|
||||
Name: "foo.txt",
|
||||
Method: Deflate,
|
||||
Extra: []byte{zip64ExtraId & 0xFF, zip64ExtraId >> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted
|
||||
Extra: []byte{zip64ExtraID & 0xFF, zip64ExtraID >> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted
|
||||
}
|
||||
testValidHeader(&h, t)
|
||||
}
|
||||
|
|
|
@ -62,6 +62,9 @@ func NewReader(rd io.Reader) *Reader {
|
|||
return NewReaderSize(rd, defaultBufSize)
|
||||
}
|
||||
|
||||
// Size returns the size of the underlying buffer in bytes.
|
||||
func (r *Reader) Size() int { return len(r.buf) }
|
||||
|
||||
// Reset discards any buffered data, resets all state, and switches
|
||||
// the buffered reader to read from r.
|
||||
func (b *Reader) Reset(r io.Reader) {
|
||||
|
@ -548,6 +551,9 @@ func NewWriter(w io.Writer) *Writer {
|
|||
return NewWriterSize(w, defaultBufSize)
|
||||
}
|
||||
|
||||
// Size returns the size of the underlying buffer in bytes.
|
||||
func (b *Writer) Size() int { return len(b.buf) }
|
||||
|
||||
// Reset discards any unflushed buffered data, clears any error, and
|
||||
// resets b to write its output to w.
|
||||
func (b *Writer) Reset(w io.Writer) {
|
||||
|
|
|
@ -1418,6 +1418,24 @@ func TestReaderDiscard(t *testing.T) {
|
|||
|
||||
}
|
||||
|
||||
func TestReaderSize(t *testing.T) {
|
||||
if got, want := NewReader(nil).Size(), DefaultBufSize; got != want {
|
||||
t.Errorf("NewReader's Reader.Size = %d; want %d", got, want)
|
||||
}
|
||||
if got, want := NewReaderSize(nil, 1234).Size(), 1234; got != want {
|
||||
t.Errorf("NewReaderSize's Reader.Size = %d; want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterSize(t *testing.T) {
|
||||
if got, want := NewWriter(nil).Size(), DefaultBufSize; got != want {
|
||||
t.Errorf("NewWriter's Writer.Size = %d; want %d", got, want)
|
||||
}
|
||||
if got, want := NewWriterSize(nil, 1234).Size(), 1234; got != want {
|
||||
t.Errorf("NewWriterSize's Writer.Size = %d; want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// An onlyReader only implements io.Reader, no matter what other methods the underlying implementation may have.
|
||||
type onlyReader struct {
|
||||
io.Reader
|
||||
|
|
|
@ -11,6 +11,8 @@ import (
|
|||
|
||||
var IsSpace = isSpace
|
||||
|
||||
const DefaultBufSize = defaultBufSize
|
||||
|
||||
func (s *Scanner) MaxTokenSize(n int) {
|
||||
if n < utf8.UTFMax || n > 1e9 {
|
||||
panic("bad max token size")
|
||||
|
|
|
@ -123,8 +123,9 @@ var ErrFinalToken = errors.New("final token")
|
|||
// After Scan returns false, the Err method will return any error that
|
||||
// occurred during scanning, except that if it was io.EOF, Err
|
||||
// will return nil.
|
||||
// Scan panics if the split function returns 100 empty tokens without
|
||||
// advancing the input. This is a common error mode for scanners.
|
||||
// Scan panics if the split function returns too many empty
|
||||
// tokens without advancing the input. This is a common error mode for
|
||||
// scanners.
|
||||
func (s *Scanner) Scan() bool {
|
||||
if s.done {
|
||||
return false
|
||||
|
@ -156,8 +157,8 @@ func (s *Scanner) Scan() bool {
|
|||
} else {
|
||||
// Returning tokens not advancing input at EOF.
|
||||
s.empties++
|
||||
if s.empties > 100 {
|
||||
panic("bufio.Scan: 100 empty tokens without progressing")
|
||||
if s.empties > maxConsecutiveEmptyReads {
|
||||
panic("bufio.Scan: too many empty tokens without progressing")
|
||||
}
|
||||
}
|
||||
return true
|
||||
|
|
|
@ -171,8 +171,9 @@ func cap(v Type) int
|
|||
// Slice: The size specifies the length. The capacity of the slice is
|
||||
// equal to its length. A second integer argument may be provided to
|
||||
// specify a different capacity; it must be no smaller than the
|
||||
// length, so make([]int, 0, 10) allocates a slice of length 0 and
|
||||
// capacity 10.
|
||||
// length. For example, make([]int, 0, 10) allocates an underlying array
|
||||
// of size 10 and returns a slice of length 0 and capacity 10 that is
|
||||
// backed by this underlying array.
|
||||
// Map: An empty map is allocated with enough space to hold the
|
||||
// specified number of elements. The size may be omitted, in which case
|
||||
// a small starting size is allocated.
|
||||
|
|
|
@ -0,0 +1,84 @@
|
|||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
//
|
||||
// +build linux
|
||||
|
||||
package bytes_test
|
||||
|
||||
import (
|
||||
. "bytes"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// This file tests the situation where byte operations are checking
|
||||
// data very near to a page boundary. We want to make sure those
|
||||
// operations do not read across the boundary and cause a page
|
||||
// fault where they shouldn't.
|
||||
|
||||
// These tests run only on linux. The code being tested is
|
||||
// not OS-specific, so it does not need to be tested on all
|
||||
// operating systems.
|
||||
|
||||
// dangerousSlice returns a slice which is immediately
|
||||
// preceded and followed by a faulting page.
|
||||
func dangerousSlice(t *testing.T) []byte {
|
||||
pagesize := syscall.Getpagesize()
|
||||
b, err := syscall.Mmap(0, 0, 3*pagesize, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANONYMOUS|syscall.MAP_PRIVATE)
|
||||
if err != nil {
|
||||
t.Fatalf("mmap failed %s", err)
|
||||
}
|
||||
err = syscall.Mprotect(b[:pagesize], syscall.PROT_NONE)
|
||||
if err != nil {
|
||||
t.Fatalf("mprotect low failed %s\n", err)
|
||||
}
|
||||
err = syscall.Mprotect(b[2*pagesize:], syscall.PROT_NONE)
|
||||
if err != nil {
|
||||
t.Fatalf("mprotect high failed %s\n", err)
|
||||
}
|
||||
return b[pagesize : 2*pagesize]
|
||||
}
|
||||
|
||||
func TestEqualNearPageBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
b := dangerousSlice(t)
|
||||
for i := range b {
|
||||
b[i] = 'A'
|
||||
}
|
||||
for i := 0; i <= len(b); i++ {
|
||||
Equal(b[:i], b[len(b)-i:])
|
||||
Equal(b[len(b)-i:], b[:i])
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexByteNearPageBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
b := dangerousSlice(t)
|
||||
for i := range b {
|
||||
idx := IndexByte(b[i:], 1)
|
||||
if idx != -1 {
|
||||
t.Fatalf("IndexByte(b[%d:])=%d, want -1\n", i, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexNearPageBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
var q [64]byte
|
||||
b := dangerousSlice(t)
|
||||
if len(b) > 256 {
|
||||
// Only worry about when we're near the end of a page.
|
||||
b = b[len(b)-256:]
|
||||
}
|
||||
for j := 1; j < len(q); j++ {
|
||||
q[j-1] = 1 // difference is only found on the last byte
|
||||
for i := range b {
|
||||
idx := Index(b[i:], q[:j])
|
||||
if idx != -1 {
|
||||
t.Fatalf("Index(b[%d:], q[:%d])=%d, want -1\n", i, j, idx)
|
||||
}
|
||||
}
|
||||
q[j-1] = 0
|
||||
}
|
||||
}
|
|
@ -15,34 +15,37 @@ import (
|
|||
// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
|
||||
// The zero value for Buffer is an empty buffer ready to use.
|
||||
type Buffer struct {
|
||||
buf []byte // contents are the bytes buf[off : len(buf)]
|
||||
off int // read at &buf[off], write at &buf[len(buf)]
|
||||
lastRead readOp // last read operation, so that Unread* can work correctly.
|
||||
// FIXME: lastRead can fit in a single byte
|
||||
buf []byte // contents are the bytes buf[off : len(buf)]
|
||||
off int // read at &buf[off], write at &buf[len(buf)]
|
||||
bootstrap [64]byte // memory to hold first slice; helps small buffers avoid allocation.
|
||||
lastRead readOp // last read operation, so that Unread* can work correctly.
|
||||
|
||||
// memory to hold first slice; helps small buffers avoid allocation.
|
||||
// FIXME: it would be advisable to align Buffer to cachelines to avoid false
|
||||
// sharing.
|
||||
bootstrap [64]byte
|
||||
}
|
||||
|
||||
// The readOp constants describe the last action performed on
|
||||
// the buffer, so that UnreadRune and UnreadByte can check for
|
||||
// invalid usage. opReadRuneX constants are chosen such that
|
||||
// converted to int they correspond to the rune size that was read.
|
||||
type readOp int
|
||||
type readOp int8
|
||||
|
||||
// Don't use iota for these, as the values need to correspond with the
|
||||
// names and comments, which is easier to see when being explicit.
|
||||
const (
|
||||
opRead readOp = -1 // Any other read operation.
|
||||
opInvalid = 0 // Non-read operation.
|
||||
opReadRune1 = 1 // Read rune of size 1.
|
||||
opReadRune2 = 2 // Read rune of size 2.
|
||||
opReadRune3 = 3 // Read rune of size 3.
|
||||
opReadRune4 = 4 // Read rune of size 4.
|
||||
opInvalid readOp = 0 // Non-read operation.
|
||||
opReadRune1 readOp = 1 // Read rune of size 1.
|
||||
opReadRune2 readOp = 2 // Read rune of size 2.
|
||||
opReadRune3 readOp = 3 // Read rune of size 3.
|
||||
opReadRune4 readOp = 4 // Read rune of size 4.
|
||||
)
|
||||
|
||||
// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
|
||||
var ErrTooLarge = errors.New("bytes.Buffer: too large")
|
||||
var errNegativeRead = errors.New("bytes.Buffer: reader returned negative count from Read")
|
||||
|
||||
const maxInt = int(^uint(0) >> 1)
|
||||
|
||||
// Bytes returns a slice of length b.Len() holding the unread portion of the buffer.
|
||||
// The slice is valid for use only until the next buffer modification (that is,
|
||||
|
@ -53,6 +56,8 @@ func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }
|
|||
|
||||
// String returns the contents of the unread portion of the buffer
|
||||
// as a string. If the Buffer is a nil pointer, it returns "<nil>".
|
||||
//
|
||||
// To build strings more efficiently, see the strings.Builder type.
|
||||
func (b *Buffer) String() string {
|
||||
if b == nil {
|
||||
// Special case, useful in debugging.
|
||||
|
@ -61,6 +66,9 @@ func (b *Buffer) String() string {
|
|||
return string(b.buf[b.off:])
|
||||
}
|
||||
|
||||
// empty returns whether the unread portion of the buffer is empty.
|
||||
func (b *Buffer) empty() bool { return len(b.buf) <= b.off }
|
||||
|
||||
// Len returns the number of bytes of the unread portion of the buffer;
|
||||
// b.Len() == len(b.Bytes()).
|
||||
func (b *Buffer) Len() int { return len(b.buf) - b.off }
|
||||
|
@ -81,7 +89,7 @@ func (b *Buffer) Truncate(n int) {
|
|||
if n < 0 || n > b.Len() {
|
||||
panic("bytes.Buffer: truncation out of range")
|
||||
}
|
||||
b.buf = b.buf[0 : b.off+n]
|
||||
b.buf = b.buf[:b.off+n]
|
||||
}
|
||||
|
||||
// Reset resets the buffer to be empty,
|
||||
|
@ -97,7 +105,7 @@ func (b *Buffer) Reset() {
|
|||
// internal buffer only needs to be resliced.
|
||||
// It returns the index where bytes should be written and whether it succeeded.
|
||||
func (b *Buffer) tryGrowByReslice(n int) (int, bool) {
|
||||
if l := len(b.buf); l+n <= cap(b.buf) {
|
||||
if l := len(b.buf); n <= cap(b.buf)-l {
|
||||
b.buf = b.buf[:l+n]
|
||||
return l, true
|
||||
}
|
||||
|
@ -122,15 +130,18 @@ func (b *Buffer) grow(n int) int {
|
|||
b.buf = b.bootstrap[:n]
|
||||
return 0
|
||||
}
|
||||
if m+n <= cap(b.buf)/2 {
|
||||
c := cap(b.buf)
|
||||
if n <= c/2-m {
|
||||
// We can slide things down instead of allocating a new
|
||||
// slice. We only need m+n <= cap(b.buf) to slide, but
|
||||
// slice. We only need m+n <= c to slide, but
|
||||
// we instead let capacity get twice as large so we
|
||||
// don't spend all our time copying.
|
||||
copy(b.buf[:], b.buf[b.off:])
|
||||
copy(b.buf, b.buf[b.off:])
|
||||
} else if c > maxInt-c-n {
|
||||
panic(ErrTooLarge)
|
||||
} else {
|
||||
// Not enough space anywhere, we need to allocate.
|
||||
buf := makeSlice(2*cap(b.buf) + n)
|
||||
buf := makeSlice(2*c + n)
|
||||
copy(buf, b.buf[b.off:])
|
||||
b.buf = buf
|
||||
}
|
||||
|
@ -150,7 +161,7 @@ func (b *Buffer) Grow(n int) {
|
|||
panic("bytes.Buffer.Grow: negative count")
|
||||
}
|
||||
m := b.grow(n)
|
||||
b.buf = b.buf[0:m]
|
||||
b.buf = b.buf[:m]
|
||||
}
|
||||
|
||||
// Write appends the contents of p to the buffer, growing the buffer as
|
||||
|
@ -189,34 +200,22 @@ const MinRead = 512
|
|||
// buffer becomes too large, ReadFrom will panic with ErrTooLarge.
|
||||
func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
|
||||
b.lastRead = opInvalid
|
||||
// If buffer is empty, reset to recover space.
|
||||
if b.off >= len(b.buf) {
|
||||
b.Reset()
|
||||
}
|
||||
for {
|
||||
if free := cap(b.buf) - len(b.buf); free < MinRead {
|
||||
// not enough space at end
|
||||
newBuf := b.buf
|
||||
if b.off+free < MinRead {
|
||||
// not enough space using beginning of buffer;
|
||||
// double buffer capacity
|
||||
newBuf = makeSlice(2*cap(b.buf) + MinRead)
|
||||
}
|
||||
copy(newBuf, b.buf[b.off:])
|
||||
b.buf = newBuf[:len(b.buf)-b.off]
|
||||
b.off = 0
|
||||
i := b.grow(MinRead)
|
||||
m, e := r.Read(b.buf[i:cap(b.buf)])
|
||||
if m < 0 {
|
||||
panic(errNegativeRead)
|
||||
}
|
||||
m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
|
||||
b.buf = b.buf[0 : len(b.buf)+m]
|
||||
|
||||
b.buf = b.buf[:i+m]
|
||||
n += int64(m)
|
||||
if e == io.EOF {
|
||||
break
|
||||
return n, nil // e is EOF, so return nil explicitly
|
||||
}
|
||||
if e != nil {
|
||||
return n, e
|
||||
}
|
||||
}
|
||||
return n, nil // err is EOF, so return nil explicitly
|
||||
}
|
||||
|
||||
// makeSlice allocates a slice of size n. If the allocation fails, it panics
|
||||
|
@ -237,8 +236,7 @@ func makeSlice(n int) []byte {
|
|||
// encountered during the write is also returned.
|
||||
func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
|
||||
b.lastRead = opInvalid
|
||||
if b.off < len(b.buf) {
|
||||
nBytes := b.Len()
|
||||
if nBytes := b.Len(); nBytes > 0 {
|
||||
m, e := w.Write(b.buf[b.off:])
|
||||
if m > nBytes {
|
||||
panic("bytes.Buffer.WriteTo: invalid Write count")
|
||||
|
@ -256,7 +254,7 @@ func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
|
|||
}
|
||||
// Buffer is now empty; reset.
|
||||
b.Reset()
|
||||
return
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// WriteByte appends the byte c to the buffer, growing the buffer as needed.
|
||||
|
@ -298,11 +296,11 @@ func (b *Buffer) WriteRune(r rune) (n int, err error) {
|
|||
// otherwise it is nil.
|
||||
func (b *Buffer) Read(p []byte) (n int, err error) {
|
||||
b.lastRead = opInvalid
|
||||
if b.off >= len(b.buf) {
|
||||
if b.empty() {
|
||||
// Buffer is empty, reset to recover space.
|
||||
b.Reset()
|
||||
if len(p) == 0 {
|
||||
return
|
||||
return 0, nil
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
@ -311,7 +309,7 @@ func (b *Buffer) Read(p []byte) (n int, err error) {
|
|||
if n > 0 {
|
||||
b.lastRead = opRead
|
||||
}
|
||||
return
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Next returns a slice containing the next n bytes from the buffer,
|
||||
|
@ -335,8 +333,7 @@ func (b *Buffer) Next(n int) []byte {
|
|||
// ReadByte reads and returns the next byte from the buffer.
|
||||
// If no byte is available, it returns error io.EOF.
|
||||
func (b *Buffer) ReadByte() (byte, error) {
|
||||
b.lastRead = opInvalid
|
||||
if b.off >= len(b.buf) {
|
||||
if b.empty() {
|
||||
// Buffer is empty, reset to recover space.
|
||||
b.Reset()
|
||||
return 0, io.EOF
|
||||
|
@ -353,8 +350,7 @@ func (b *Buffer) ReadByte() (byte, error) {
|
|||
// If the bytes are an erroneous UTF-8 encoding, it
|
||||
// consumes one byte and returns U+FFFD, 1.
|
||||
func (b *Buffer) ReadRune() (r rune, size int, err error) {
|
||||
b.lastRead = opInvalid
|
||||
if b.off >= len(b.buf) {
|
||||
if b.empty() {
|
||||
// Buffer is empty, reset to recover space.
|
||||
b.Reset()
|
||||
return 0, 0, io.EOF
|
||||
|
@ -413,7 +409,7 @@ func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) {
|
|||
// return a copy of slice. The buffer's backing array may
|
||||
// be overwritten by later calls.
|
||||
line = append(line, slice...)
|
||||
return
|
||||
return line, err
|
||||
}
|
||||
|
||||
// readSlice is like ReadBytes but returns a reference to internal buffer data.
|
||||
|
|
|
@ -6,25 +6,27 @@ package bytes_test
|
|||
|
||||
import (
|
||||
. "bytes"
|
||||
"internal/testenv"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const N = 10000 // make this bigger for a larger (and slower) test
|
||||
var data string // test data for write tests
|
||||
var testBytes []byte // test data; same as data but as a slice.
|
||||
const N = 10000 // make this bigger for a larger (and slower) test
|
||||
var testString string // test data for write tests
|
||||
var testBytes []byte // test data; same as testString but as a slice.
|
||||
|
||||
type negativeReader struct{}
|
||||
|
||||
func (r *negativeReader) Read([]byte) (int, error) { return -1, nil }
|
||||
|
||||
func init() {
|
||||
testBytes = make([]byte, N)
|
||||
for i := 0; i < N; i++ {
|
||||
testBytes[i] = 'a' + byte(i%26)
|
||||
}
|
||||
data = string(testBytes)
|
||||
testString = string(testBytes)
|
||||
}
|
||||
|
||||
// Verify that contents of buf match the string s.
|
||||
|
@ -88,12 +90,12 @@ func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub
|
|||
|
||||
func TestNewBuffer(t *testing.T) {
|
||||
buf := NewBuffer(testBytes)
|
||||
check(t, "NewBuffer", buf, data)
|
||||
check(t, "NewBuffer", buf, testString)
|
||||
}
|
||||
|
||||
func TestNewBufferString(t *testing.T) {
|
||||
buf := NewBufferString(data)
|
||||
check(t, "NewBufferString", buf, data)
|
||||
buf := NewBufferString(testString)
|
||||
check(t, "NewBufferString", buf, testString)
|
||||
}
|
||||
|
||||
// Empty buf through repeated reads into fub.
|
||||
|
@ -128,7 +130,7 @@ func TestBasicOperations(t *testing.T) {
|
|||
buf.Truncate(0)
|
||||
check(t, "TestBasicOperations (3)", &buf, "")
|
||||
|
||||
n, err := buf.Write([]byte(data[0:1]))
|
||||
n, err := buf.Write(testBytes[0:1])
|
||||
if n != 1 {
|
||||
t.Errorf("wrote 1 byte, but n == %d", n)
|
||||
}
|
||||
|
@ -137,30 +139,30 @@ func TestBasicOperations(t *testing.T) {
|
|||
}
|
||||
check(t, "TestBasicOperations (4)", &buf, "a")
|
||||
|
||||
buf.WriteByte(data[1])
|
||||
buf.WriteByte(testString[1])
|
||||
check(t, "TestBasicOperations (5)", &buf, "ab")
|
||||
|
||||
n, err = buf.Write([]byte(data[2:26]))
|
||||
n, err = buf.Write(testBytes[2:26])
|
||||
if n != 24 {
|
||||
t.Errorf("wrote 25 bytes, but n == %d", n)
|
||||
t.Errorf("wrote 24 bytes, but n == %d", n)
|
||||
}
|
||||
check(t, "TestBasicOperations (6)", &buf, string(data[0:26]))
|
||||
check(t, "TestBasicOperations (6)", &buf, testString[0:26])
|
||||
|
||||
buf.Truncate(26)
|
||||
check(t, "TestBasicOperations (7)", &buf, string(data[0:26]))
|
||||
check(t, "TestBasicOperations (7)", &buf, testString[0:26])
|
||||
|
||||
buf.Truncate(20)
|
||||
check(t, "TestBasicOperations (8)", &buf, string(data[0:20]))
|
||||
check(t, "TestBasicOperations (8)", &buf, testString[0:20])
|
||||
|
||||
empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5))
|
||||
empty(t, "TestBasicOperations (9)", &buf, testString[0:20], make([]byte, 5))
|
||||
empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100))
|
||||
|
||||
buf.WriteByte(data[1])
|
||||
buf.WriteByte(testString[1])
|
||||
c, err := buf.ReadByte()
|
||||
if err != nil {
|
||||
t.Error("ReadByte unexpected eof")
|
||||
}
|
||||
if c != data[1] {
|
||||
if c != testString[1] {
|
||||
t.Errorf("ReadByte wrong value c=%v", c)
|
||||
}
|
||||
c, err = buf.ReadByte()
|
||||
|
@ -177,8 +179,8 @@ func TestLargeStringWrites(t *testing.T) {
|
|||
limit = 9
|
||||
}
|
||||
for i := 3; i < limit; i += 3 {
|
||||
s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, data)
|
||||
empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(data)/i))
|
||||
s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, testString)
|
||||
empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(testString)/i))
|
||||
}
|
||||
check(t, "TestLargeStringWrites (3)", &buf, "")
|
||||
}
|
||||
|
@ -191,7 +193,7 @@ func TestLargeByteWrites(t *testing.T) {
|
|||
}
|
||||
for i := 3; i < limit; i += 3 {
|
||||
s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, testBytes)
|
||||
empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i))
|
||||
empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(testString)/i))
|
||||
}
|
||||
check(t, "TestLargeByteWrites (3)", &buf, "")
|
||||
}
|
||||
|
@ -199,8 +201,8 @@ func TestLargeByteWrites(t *testing.T) {
|
|||
func TestLargeStringReads(t *testing.T) {
|
||||
var buf Buffer
|
||||
for i := 3; i < 30; i += 3 {
|
||||
s := fillString(t, "TestLargeReads (1)", &buf, "", 5, data[0:len(data)/i])
|
||||
empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data)))
|
||||
s := fillString(t, "TestLargeReads (1)", &buf, "", 5, testString[0:len(testString)/i])
|
||||
empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(testString)))
|
||||
}
|
||||
check(t, "TestLargeStringReads (3)", &buf, "")
|
||||
}
|
||||
|
@ -209,7 +211,7 @@ func TestLargeByteReads(t *testing.T) {
|
|||
var buf Buffer
|
||||
for i := 3; i < 30; i += 3 {
|
||||
s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
|
||||
empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data)))
|
||||
empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(testString)))
|
||||
}
|
||||
check(t, "TestLargeByteReads (3)", &buf, "")
|
||||
}
|
||||
|
@ -218,14 +220,14 @@ func TestMixedReadsAndWrites(t *testing.T) {
|
|||
var buf Buffer
|
||||
s := ""
|
||||
for i := 0; i < 50; i++ {
|
||||
wlen := rand.Intn(len(data))
|
||||
wlen := rand.Intn(len(testString))
|
||||
if i%2 == 0 {
|
||||
s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, data[0:wlen])
|
||||
s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testString[0:wlen])
|
||||
} else {
|
||||
s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testBytes[0:wlen])
|
||||
}
|
||||
|
||||
rlen := rand.Intn(len(data))
|
||||
rlen := rand.Intn(len(testString))
|
||||
fub := make([]byte, rlen)
|
||||
n, _ := buf.Read(fub)
|
||||
s = s[n:]
|
||||
|
@ -263,17 +265,37 @@ func TestReadFrom(t *testing.T) {
|
|||
s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
|
||||
var b Buffer
|
||||
b.ReadFrom(&buf)
|
||||
empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data)))
|
||||
empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(testString)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFromNegativeReader(t *testing.T) {
|
||||
var b Buffer
|
||||
defer func() {
|
||||
switch err := recover().(type) {
|
||||
case nil:
|
||||
t.Fatal("bytes.Buffer.ReadFrom didn't panic")
|
||||
case error:
|
||||
// this is the error string of errNegativeRead
|
||||
wantError := "bytes.Buffer: reader returned negative count from Read"
|
||||
if err.Error() != wantError {
|
||||
t.Fatalf("recovered panic: got %v, want %v", err.Error(), wantError)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected panic value: %#v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
b.ReadFrom(new(negativeReader))
|
||||
}
|
||||
|
||||
func TestWriteTo(t *testing.T) {
|
||||
var buf Buffer
|
||||
for i := 3; i < 30; i += 3 {
|
||||
s := fillBytes(t, "TestWriteTo (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
|
||||
var b Buffer
|
||||
buf.WriteTo(&b)
|
||||
empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(data)))
|
||||
empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(testString)))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -473,6 +495,18 @@ func TestGrow(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestGrowOverflow(t *testing.T) {
|
||||
defer func() {
|
||||
if err := recover(); err != ErrTooLarge {
|
||||
t.Errorf("after too-large Grow, recover() = %v; want %v", err, ErrTooLarge)
|
||||
}
|
||||
}()
|
||||
|
||||
buf := NewBuffer(make([]byte, 1))
|
||||
const maxInt = int(^uint(0) >> 1)
|
||||
buf.Grow(maxInt)
|
||||
}
|
||||
|
||||
// Was a bug: used to give EOF reading empty slice at EOF.
|
||||
func TestReadEmptyAtEOF(t *testing.T) {
|
||||
b := new(Buffer)
|
||||
|
@ -548,26 +582,6 @@ func TestBufferGrowth(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Test that tryGrowByReslice is inlined.
|
||||
// Only execute on "linux-amd64" builder in order to avoid breakage.
|
||||
func TestTryGrowByResliceInlined(t *testing.T) {
|
||||
targetBuilder := "linux-amd64"
|
||||
if testenv.Builder() != targetBuilder {
|
||||
t.Skipf("%q gets executed on %q builder only", t.Name(), targetBuilder)
|
||||
}
|
||||
t.Parallel()
|
||||
goBin := testenv.GoToolPath(t)
|
||||
out, err := exec.Command(goBin, "tool", "nm", goBin).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("go tool nm: %v: %s", err, out)
|
||||
}
|
||||
// Verify this doesn't exist:
|
||||
sym := "bytes.(*Buffer).tryGrowByReslice"
|
||||
if Contains(out, []byte(sym)) {
|
||||
t.Errorf("found symbol %q in cmd/go, but should be inlined", sym)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkWriteByte(b *testing.B) {
|
||||
const n = 4 << 10
|
||||
b.SetBytes(n)
|
||||
|
|
|
@ -39,7 +39,7 @@ func explode(s []byte, n int) [][]byte {
|
|||
break
|
||||
}
|
||||
_, size = utf8.DecodeRune(s)
|
||||
a[na] = s[0:size]
|
||||
a[na] = s[0:size:size]
|
||||
s = s[size:]
|
||||
na++
|
||||
}
|
||||
|
@ -68,12 +68,12 @@ func Contains(b, subslice []byte) bool {
|
|||
return Index(b, subslice) != -1
|
||||
}
|
||||
|
||||
// ContainsAny reports whether any of the UTF-8-encoded Unicode code points in chars are within b.
|
||||
// ContainsAny reports whether any of the UTF-8-encoded code points in chars are within b.
|
||||
func ContainsAny(b []byte, chars string) bool {
|
||||
return IndexAny(b, chars) >= 0
|
||||
}
|
||||
|
||||
// ContainsRune reports whether the Unicode code point r is within b.
|
||||
// ContainsRune reports whether the rune is contained in the UTF-8-encoded byte slice b.
|
||||
func ContainsRune(b []byte, r rune) bool {
|
||||
return IndexRune(b, r) >= 0
|
||||
}
|
||||
|
@ -112,7 +112,7 @@ func LastIndexByte(s []byte, c byte) int {
|
|||
return -1
|
||||
}
|
||||
|
||||
// IndexRune interprets s as a sequence of UTF-8-encoded Unicode code points.
|
||||
// IndexRune interprets s as a sequence of UTF-8-encoded code points.
|
||||
// It returns the byte index of the first occurrence in s of the given rune.
|
||||
// It returns -1 if rune is not present in s.
|
||||
// If r is utf8.RuneError, it returns the first instance of any
|
||||
|
@ -144,30 +144,32 @@ func IndexRune(s []byte, r rune) int {
|
|||
// code points in chars. It returns -1 if chars is empty or if there is no code
|
||||
// point in common.
|
||||
func IndexAny(s []byte, chars string) int {
|
||||
if len(chars) > 0 {
|
||||
if len(s) > 8 {
|
||||
if as, isASCII := makeASCIISet(chars); isASCII {
|
||||
for i, c := range s {
|
||||
if as.contains(c) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
}
|
||||
var width int
|
||||
for i := 0; i < len(s); i += width {
|
||||
r := rune(s[i])
|
||||
if r < utf8.RuneSelf {
|
||||
width = 1
|
||||
} else {
|
||||
r, width = utf8.DecodeRune(s[i:])
|
||||
}
|
||||
for _, ch := range chars {
|
||||
if r == ch {
|
||||
if chars == "" {
|
||||
// Avoid scanning all of s.
|
||||
return -1
|
||||
}
|
||||
if len(s) > 8 {
|
||||
if as, isASCII := makeASCIISet(chars); isASCII {
|
||||
for i, c := range s {
|
||||
if as.contains(c) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
}
|
||||
var width int
|
||||
for i := 0; i < len(s); i += width {
|
||||
r := rune(s[i])
|
||||
if r < utf8.RuneSelf {
|
||||
width = 1
|
||||
} else {
|
||||
r, width = utf8.DecodeRune(s[i:])
|
||||
}
|
||||
for _, ch := range chars {
|
||||
if r == ch {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
|
@ -178,25 +180,27 @@ func IndexAny(s []byte, chars string) int {
|
|||
// the Unicode code points in chars. It returns -1 if chars is empty or if
|
||||
// there is no code point in common.
|
||||
func LastIndexAny(s []byte, chars string) int {
|
||||
if len(chars) > 0 {
|
||||
if len(s) > 8 {
|
||||
if as, isASCII := makeASCIISet(chars); isASCII {
|
||||
for i := len(s) - 1; i >= 0; i-- {
|
||||
if as.contains(s[i]) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
}
|
||||
for i := len(s); i > 0; {
|
||||
r, size := utf8.DecodeLastRune(s[:i])
|
||||
i -= size
|
||||
for _, c := range chars {
|
||||
if r == c {
|
||||
if chars == "" {
|
||||
// Avoid scanning all of s.
|
||||
return -1
|
||||
}
|
||||
if len(s) > 8 {
|
||||
if as, isASCII := makeASCIISet(chars); isASCII {
|
||||
for i := len(s) - 1; i >= 0; i-- {
|
||||
if as.contains(s[i]) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
}
|
||||
for i := len(s); i > 0; {
|
||||
r, size := utf8.DecodeLastRune(s[:i])
|
||||
i -= size
|
||||
for _, c := range chars {
|
||||
if r == c {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
|
@ -223,7 +227,7 @@ func genSplit(s, sep []byte, sepSave, n int) [][]byte {
|
|||
if m < 0 {
|
||||
break
|
||||
}
|
||||
a[i] = s[:m+sepSave]
|
||||
a[i] = s[: m+sepSave : m+sepSave]
|
||||
s = s[m+len(sep):]
|
||||
i++
|
||||
}
|
||||
|
@ -265,52 +269,112 @@ func SplitAfter(s, sep []byte) [][]byte {
|
|||
return genSplit(s, sep, len(sep), -1)
|
||||
}
|
||||
|
||||
// Fields splits the slice s around each instance of one or more consecutive white space
|
||||
// characters, returning a slice of subslices of s or an empty list if s contains only white space.
|
||||
var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1}
|
||||
|
||||
// Fields interprets s as a sequence of UTF-8-encoded code points.
|
||||
// It splits the slice s around each instance of one or more consecutive white space
|
||||
// characters, as defined by unicode.IsSpace, returning a slice of subslices of s or an
|
||||
// empty slice if s contains only white space.
|
||||
func Fields(s []byte) [][]byte {
|
||||
return FieldsFunc(s, unicode.IsSpace)
|
||||
// First count the fields.
|
||||
// This is an exact count if s is ASCII, otherwise it is an approximation.
|
||||
n := 0
|
||||
wasSpace := 1
|
||||
// setBits is used to track which bits are set in the bytes of s.
|
||||
setBits := uint8(0)
|
||||
for i := 0; i < len(s); i++ {
|
||||
r := s[i]
|
||||
setBits |= r
|
||||
isSpace := int(asciiSpace[r])
|
||||
n += wasSpace & ^isSpace
|
||||
wasSpace = isSpace
|
||||
}
|
||||
|
||||
if setBits >= utf8.RuneSelf {
|
||||
// Some runes in the input slice are not ASCII.
|
||||
return FieldsFunc(s, unicode.IsSpace)
|
||||
}
|
||||
|
||||
// ASCII fast path
|
||||
a := make([][]byte, n)
|
||||
na := 0
|
||||
fieldStart := 0
|
||||
i := 0
|
||||
// Skip spaces in the front of the input.
|
||||
for i < len(s) && asciiSpace[s[i]] != 0 {
|
||||
i++
|
||||
}
|
||||
fieldStart = i
|
||||
for i < len(s) {
|
||||
if asciiSpace[s[i]] == 0 {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
a[na] = s[fieldStart:i:i]
|
||||
na++
|
||||
i++
|
||||
// Skip spaces in between fields.
|
||||
for i < len(s) && asciiSpace[s[i]] != 0 {
|
||||
i++
|
||||
}
|
||||
fieldStart = i
|
||||
}
|
||||
if fieldStart < len(s) { // Last field might end at EOF.
|
||||
a[na] = s[fieldStart:len(s):len(s)]
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// FieldsFunc interprets s as a sequence of UTF-8-encoded Unicode code points.
|
||||
// FieldsFunc interprets s as a sequence of UTF-8-encoded code points.
|
||||
// It splits the slice s at each run of code points c satisfying f(c) and
|
||||
// returns a slice of subslices of s. If all code points in s satisfy f(c), or
|
||||
// len(s) == 0, an empty slice is returned.
|
||||
// FieldsFunc makes no guarantees about the order in which it calls f(c).
|
||||
// If f does not return consistent results for a given c, FieldsFunc may crash.
|
||||
func FieldsFunc(s []byte, f func(rune) bool) [][]byte {
|
||||
n := 0
|
||||
inField := false
|
||||
// A span is used to record a slice of s of the form s[start:end].
|
||||
// The start index is inclusive and the end index is exclusive.
|
||||
type span struct {
|
||||
start int
|
||||
end int
|
||||
}
|
||||
spans := make([]span, 0, 32)
|
||||
|
||||
// Find the field start and end indices.
|
||||
wasField := false
|
||||
fromIndex := 0
|
||||
for i := 0; i < len(s); {
|
||||
r, size := utf8.DecodeRune(s[i:])
|
||||
wasInField := inField
|
||||
inField = !f(r)
|
||||
if inField && !wasInField {
|
||||
n++
|
||||
size := 1
|
||||
r := rune(s[i])
|
||||
if r >= utf8.RuneSelf {
|
||||
r, size = utf8.DecodeRune(s[i:])
|
||||
}
|
||||
if f(r) {
|
||||
if wasField {
|
||||
spans = append(spans, span{start: fromIndex, end: i})
|
||||
wasField = false
|
||||
}
|
||||
} else {
|
||||
if !wasField {
|
||||
fromIndex = i
|
||||
wasField = true
|
||||
}
|
||||
}
|
||||
i += size
|
||||
}
|
||||
|
||||
a := make([][]byte, n)
|
||||
na := 0
|
||||
fieldStart := -1
|
||||
for i := 0; i <= len(s) && na < n; {
|
||||
r, size := utf8.DecodeRune(s[i:])
|
||||
if fieldStart < 0 && size > 0 && !f(r) {
|
||||
fieldStart = i
|
||||
i += size
|
||||
continue
|
||||
}
|
||||
if fieldStart >= 0 && (size == 0 || f(r)) {
|
||||
a[na] = s[fieldStart:i]
|
||||
na++
|
||||
fieldStart = -1
|
||||
}
|
||||
if size == 0 {
|
||||
break
|
||||
}
|
||||
i += size
|
||||
// Last field might end at EOF.
|
||||
if wasField {
|
||||
spans = append(spans, span{fromIndex, len(s)})
|
||||
}
|
||||
return a[0:na]
|
||||
|
||||
// Create subslices from recorded field indices.
|
||||
a := make([][]byte, len(spans))
|
||||
for i, span := range spans {
|
||||
a[i] = s[span.start:span.end:span.end]
|
||||
}
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
// Join concatenates the elements of s to create a new byte slice. The separator
|
||||
|
@ -349,8 +413,8 @@ func HasSuffix(s, suffix []byte) bool {
|
|||
|
||||
// Map returns a copy of the byte slice s with all its characters modified
|
||||
// according to the mapping function. If mapping returns a negative value, the character is
|
||||
// dropped from the string with no replacement. The characters in s and the
|
||||
// output are interpreted as UTF-8-encoded Unicode code points.
|
||||
// dropped from the byte slice with no replacement. The characters in s and the
|
||||
// output are interpreted as UTF-8-encoded code points.
|
||||
func Map(mapping func(r rune) rune, s []byte) []byte {
|
||||
// In the worst case, the slice can grow when mapped, making
|
||||
// things unpleasant. But it's so rare we barge in assuming it's
|
||||
|
@ -408,28 +472,28 @@ func Repeat(b []byte, count int) []byte {
|
|||
return nb
|
||||
}
|
||||
|
||||
// ToUpper returns a copy of the byte slice s with all Unicode letters mapped to their upper case.
|
||||
// ToUpper treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters within it mapped to their upper case.
|
||||
func ToUpper(s []byte) []byte { return Map(unicode.ToUpper, s) }
|
||||
|
||||
// ToLower returns a copy of the byte slice s with all Unicode letters mapped to their lower case.
|
||||
// ToLower treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their lower case.
|
||||
func ToLower(s []byte) []byte { return Map(unicode.ToLower, s) }
|
||||
|
||||
// ToTitle returns a copy of the byte slice s with all Unicode letters mapped to their title case.
|
||||
// ToTitle treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their title case.
|
||||
func ToTitle(s []byte) []byte { return Map(unicode.ToTitle, s) }
|
||||
|
||||
// ToUpperSpecial returns a copy of the byte slice s with all Unicode letters mapped to their
|
||||
// ToUpperSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their
|
||||
// upper case, giving priority to the special casing rules.
|
||||
func ToUpperSpecial(c unicode.SpecialCase, s []byte) []byte {
|
||||
return Map(func(r rune) rune { return c.ToUpper(r) }, s)
|
||||
}
|
||||
|
||||
// ToLowerSpecial returns a copy of the byte slice s with all Unicode letters mapped to their
|
||||
// ToLowerSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their
|
||||
// lower case, giving priority to the special casing rules.
|
||||
func ToLowerSpecial(c unicode.SpecialCase, s []byte) []byte {
|
||||
return Map(func(r rune) rune { return c.ToLower(r) }, s)
|
||||
}
|
||||
|
||||
// ToTitleSpecial returns a copy of the byte slice s with all Unicode letters mapped to their
|
||||
// ToTitleSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their
|
||||
// title case, giving priority to the special casing rules.
|
||||
func ToTitleSpecial(c unicode.SpecialCase, s []byte) []byte {
|
||||
return Map(func(r rune) rune { return c.ToTitle(r) }, s)
|
||||
|
@ -460,8 +524,8 @@ func isSeparator(r rune) bool {
|
|||
return unicode.IsSpace(r)
|
||||
}
|
||||
|
||||
// Title returns a copy of s with all Unicode letters that begin words
|
||||
// mapped to their title case.
|
||||
// Title treats s as UTF-8-encoded bytes and returns a copy with all Unicode letters that begin
|
||||
// words mapped to their title case.
|
||||
//
|
||||
// BUG(rsc): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
|
||||
func Title(s []byte) []byte {
|
||||
|
@ -481,8 +545,8 @@ func Title(s []byte) []byte {
|
|||
s)
|
||||
}
|
||||
|
||||
// TrimLeftFunc returns a subslice of s by slicing off all leading UTF-8-encoded
|
||||
// Unicode code points c that satisfy f(c).
|
||||
// TrimLeftFunc treats s as UTF-8-encoded bytes and returns a subslice of s by slicing off
|
||||
// all leading UTF-8-encoded code points c that satisfy f(c).
|
||||
func TrimLeftFunc(s []byte, f func(r rune) bool) []byte {
|
||||
i := indexFunc(s, f, false)
|
||||
if i == -1 {
|
||||
|
@ -491,8 +555,8 @@ func TrimLeftFunc(s []byte, f func(r rune) bool) []byte {
|
|||
return s[i:]
|
||||
}
|
||||
|
||||
// TrimRightFunc returns a subslice of s by slicing off all trailing UTF-8
|
||||
// encoded Unicode code points c that satisfy f(c).
|
||||
// TrimRightFunc returns a subslice of s by slicing off all trailing
|
||||
// UTF-8-encoded code points c that satisfy f(c).
|
||||
func TrimRightFunc(s []byte, f func(r rune) bool) []byte {
|
||||
i := lastIndexFunc(s, f, false)
|
||||
if i >= 0 && s[i] >= utf8.RuneSelf {
|
||||
|
@ -505,7 +569,7 @@ func TrimRightFunc(s []byte, f func(r rune) bool) []byte {
|
|||
}
|
||||
|
||||
// TrimFunc returns a subslice of s by slicing off all leading and trailing
|
||||
// UTF-8-encoded Unicode code points c that satisfy f(c).
|
||||
// UTF-8-encoded code points c that satisfy f(c).
|
||||
func TrimFunc(s []byte, f func(r rune) bool) []byte {
|
||||
return TrimRightFunc(TrimLeftFunc(s, f), f)
|
||||
}
|
||||
|
@ -528,14 +592,14 @@ func TrimSuffix(s, suffix []byte) []byte {
|
|||
return s
|
||||
}
|
||||
|
||||
// IndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points.
|
||||
// IndexFunc interprets s as a sequence of UTF-8-encoded code points.
|
||||
// It returns the byte index in s of the first Unicode
|
||||
// code point satisfying f(c), or -1 if none do.
|
||||
func IndexFunc(s []byte, f func(r rune) bool) int {
|
||||
return indexFunc(s, f, true)
|
||||
}
|
||||
|
||||
// LastIndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points.
|
||||
// LastIndexFunc interprets s as a sequence of UTF-8-encoded code points.
|
||||
// It returns the byte index in s of the last Unicode
|
||||
// code point satisfying f(c), or -1 if none do.
|
||||
func LastIndexFunc(s []byte, f func(r rune) bool) int {
|
||||
|
@ -626,19 +690,19 @@ func makeCutsetFunc(cutset string) func(r rune) bool {
|
|||
}
|
||||
|
||||
// Trim returns a subslice of s by slicing off all leading and
|
||||
// trailing UTF-8-encoded Unicode code points contained in cutset.
|
||||
// trailing UTF-8-encoded code points contained in cutset.
|
||||
func Trim(s []byte, cutset string) []byte {
|
||||
return TrimFunc(s, makeCutsetFunc(cutset))
|
||||
}
|
||||
|
||||
// TrimLeft returns a subslice of s by slicing off all leading
|
||||
// UTF-8-encoded Unicode code points contained in cutset.
|
||||
// UTF-8-encoded code points contained in cutset.
|
||||
func TrimLeft(s []byte, cutset string) []byte {
|
||||
return TrimLeftFunc(s, makeCutsetFunc(cutset))
|
||||
}
|
||||
|
||||
// TrimRight returns a subslice of s by slicing off all trailing
|
||||
// UTF-8-encoded Unicode code points that are contained in cutset.
|
||||
// UTF-8-encoded code points that are contained in cutset.
|
||||
func TrimRight(s []byte, cutset string) []byte {
|
||||
return TrimRightFunc(s, makeCutsetFunc(cutset))
|
||||
}
|
||||
|
@ -649,7 +713,8 @@ func TrimSpace(s []byte) []byte {
|
|||
return TrimFunc(s, unicode.IsSpace)
|
||||
}
|
||||
|
||||
// Runes returns a slice of runes (Unicode code points) equivalent to s.
|
||||
// Runes interprets s as a sequence of UTF-8-encoded code points.
|
||||
// It returns a slice of runes (Unicode code points) equivalent to s.
|
||||
func Runes(s []byte) []rune {
|
||||
t := make([]rune, utf8.RuneCount(s))
|
||||
i := 0
|
||||
|
@ -758,3 +823,46 @@ func EqualFold(s, t []byte) bool {
|
|||
// One string is empty. Are both?
|
||||
return len(s) == len(t)
|
||||
}
|
||||
|
||||
func indexRabinKarp(s, sep []byte) int {
|
||||
// Rabin-Karp search
|
||||
hashsep, pow := hashStr(sep)
|
||||
n := len(sep)
|
||||
var h uint32
|
||||
for i := 0; i < n; i++ {
|
||||
h = h*primeRK + uint32(s[i])
|
||||
}
|
||||
if h == hashsep && Equal(s[:n], sep) {
|
||||
return 0
|
||||
}
|
||||
for i := n; i < len(s); {
|
||||
h *= primeRK
|
||||
h += uint32(s[i])
|
||||
h -= pow * uint32(s[i-n])
|
||||
i++
|
||||
if h == hashsep && Equal(s[i-n:i], sep) {
|
||||
return i - n
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// primeRK is the prime base used in Rabin-Karp algorithm.
|
||||
const primeRK = 16777619
|
||||
|
||||
// hashStr returns the hash and the appropriate multiplicative
|
||||
// factor for use in Rabin-Karp algorithm.
|
||||
func hashStr(sep []byte) (uint32, uint32) {
|
||||
hash := uint32(0)
|
||||
for i := 0; i < len(sep); i++ {
|
||||
hash = hash*primeRK + uint32(sep[i])
|
||||
}
|
||||
var pow, sq uint32 = 1, primeRK
|
||||
for i := len(sep); i > 0; i >>= 1 {
|
||||
if i&1 != 0 {
|
||||
pow *= sq
|
||||
}
|
||||
sq *= sq
|
||||
}
|
||||
return hash, pow
|
||||
}
|
||||
|
|
|
@ -77,52 +77,14 @@ func Index(s, sep []byte) int {
|
|||
}
|
||||
return -1
|
||||
}
|
||||
// Rabin-Karp search
|
||||
hashsep, pow := hashStr(sep)
|
||||
var h uint32
|
||||
for i := 0; i < n; i++ {
|
||||
h = h*primeRK + uint32(s[i])
|
||||
}
|
||||
if h == hashsep && Equal(s[:n], sep) {
|
||||
return 0
|
||||
}
|
||||
for i := n; i < len(s); {
|
||||
h *= primeRK
|
||||
h += uint32(s[i])
|
||||
h -= pow * uint32(s[i-n])
|
||||
i++
|
||||
if h == hashsep && Equal(s[i-n:i], sep) {
|
||||
return i - n
|
||||
}
|
||||
}
|
||||
return -1
|
||||
return indexRabinKarp(s, sep)
|
||||
}
|
||||
|
||||
// Count counts the number of non-overlapping instances of sep in s.
|
||||
// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s.
|
||||
// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
|
||||
func Count(s, sep []byte) int {
|
||||
if len(sep) == 1 && cpu.X86.HasPOPCNT {
|
||||
return countByte(s, sep[0])
|
||||
}
|
||||
return countGeneric(s, sep)
|
||||
}
|
||||
|
||||
// primeRK is the prime base used in Rabin-Karp algorithm.
|
||||
const primeRK = 16777619
|
||||
|
||||
// hashStr returns the hash and the appropriate multiplicative
|
||||
// factor for use in Rabin-Karp algorithm.
|
||||
func hashStr(sep []byte) (uint32, uint32) {
|
||||
hash := uint32(0)
|
||||
for i := 0; i < len(sep); i++ {
|
||||
hash = hash*primeRK + uint32(sep[i])
|
||||
}
|
||||
var pow, sq uint32 = 1, primeRK
|
||||
for i := len(sep); i > 0; i >>= 1 {
|
||||
if i&1 != 0 {
|
||||
pow *= sq
|
||||
}
|
||||
sq *= sq
|
||||
}
|
||||
return hash, pow
|
||||
}
|
||||
|
|
|
@ -0,0 +1,70 @@
|
|||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package bytes
|
||||
|
||||
func countByte(s []byte, c byte) int // bytes_arm64.s
|
||||
|
||||
// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s.
|
||||
func Index(s, sep []byte) int {
|
||||
n := len(sep)
|
||||
switch {
|
||||
case n == 0:
|
||||
return 0
|
||||
case n == 1:
|
||||
return IndexByte(s, sep[0])
|
||||
case n == len(s):
|
||||
if Equal(sep, s) {
|
||||
return 0
|
||||
}
|
||||
return -1
|
||||
case n > len(s):
|
||||
return -1
|
||||
}
|
||||
c := sep[0]
|
||||
i := 0
|
||||
fails := 0
|
||||
t := s[:len(s)-n+1]
|
||||
for i < len(t) {
|
||||
if t[i] != c {
|
||||
o := IndexByte(t[i:], c)
|
||||
if o < 0 {
|
||||
break
|
||||
}
|
||||
i += o
|
||||
}
|
||||
if Equal(s[i:i+n], sep) {
|
||||
return i
|
||||
}
|
||||
i++
|
||||
fails++
|
||||
if fails >= 4+i>>4 && i < len(t) {
|
||||
// Give up on IndexByte, it isn't skipping ahead
|
||||
// far enough to be better than Rabin-Karp.
|
||||
// Experiments (using IndexPeriodic) suggest
|
||||
// the cutover is about 16 byte skips.
|
||||
// TODO: if large prefixes of sep are matching
|
||||
// we should cutover at even larger average skips,
|
||||
// because Equal becomes that much more expensive.
|
||||
// This code does not take that effect into account.
|
||||
j := indexRabinKarp(s[i:], sep)
|
||||
if j < 0 {
|
||||
return -1
|
||||
}
|
||||
return i + j
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// Count counts the number of non-overlapping instances of sep in s.
|
||||
// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
|
||||
func Count(s, sep []byte) int {
|
||||
if len(sep) == 1 {
|
||||
return countByte(s, sep[0])
|
||||
}
|
||||
return countGeneric(s, sep)
|
||||
}
|
|
@ -2,27 +2,29 @@
|
|||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// -build !amd64,!s390x
|
||||
// -build !amd64,!s390x,!arm64
|
||||
|
||||
package bytes
|
||||
|
||||
// TODO: implements short string optimization on non amd64 platforms
|
||||
// and get rid of bytes_amd64.go
|
||||
|
||||
// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s.
|
||||
func Index(s, sep []byte) int {
|
||||
n := len(sep)
|
||||
if n == 0 {
|
||||
switch {
|
||||
case n == 0:
|
||||
return 0
|
||||
}
|
||||
if n > len(s) {
|
||||
case n == 1:
|
||||
return IndexByte(s, sep[0])
|
||||
case n == len(s):
|
||||
if Equal(sep, s) {
|
||||
return 0
|
||||
}
|
||||
return -1
|
||||
case n > len(s):
|
||||
return -1
|
||||
}
|
||||
c := sep[0]
|
||||
if n == 1 {
|
||||
return IndexByte(s, c)
|
||||
}
|
||||
i := 0
|
||||
fails := 0
|
||||
t := s[:len(s)-n+1]
|
||||
for i < len(t) {
|
||||
if t[i] != c {
|
||||
|
@ -36,12 +38,28 @@ func Index(s, sep []byte) int {
|
|||
return i
|
||||
}
|
||||
i++
|
||||
fails++
|
||||
if fails >= 4+i>>4 && i < len(t) {
|
||||
// Give up on IndexByte, it isn't skipping ahead
|
||||
// far enough to be better than Rabin-Karp.
|
||||
// Experiments (using IndexPeriodic) suggest
|
||||
// the cutover is about 16 byte skips.
|
||||
// TODO: if large prefixes of sep are matching
|
||||
// we should cutover at even larger average skips,
|
||||
// because Equal becomes that much more expensive.
|
||||
// This code does not take that effect into account.
|
||||
j := indexRabinKarp(s[i:], sep)
|
||||
if j < 0 {
|
||||
return -1
|
||||
}
|
||||
return i + j
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// Count counts the number of non-overlapping instances of sep in s.
|
||||
// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s.
|
||||
// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
|
||||
func Count(s, sep []byte) int {
|
||||
return countGeneric(s, sep)
|
||||
}
|
||||
|
|
|
@ -78,49 +78,11 @@ func Index(s, sep []byte) int {
|
|||
}
|
||||
return -1
|
||||
}
|
||||
// Rabin-Karp search
|
||||
hashsep, pow := hashStr(sep)
|
||||
var h uint32
|
||||
for i := 0; i < n; i++ {
|
||||
h = h*primeRK + uint32(s[i])
|
||||
}
|
||||
if h == hashsep && Equal(s[:n], sep) {
|
||||
return 0
|
||||
}
|
||||
for i := n; i < len(s); {
|
||||
h *= primeRK
|
||||
h += uint32(s[i])
|
||||
h -= pow * uint32(s[i-n])
|
||||
i++
|
||||
if h == hashsep && Equal(s[i-n:i], sep) {
|
||||
return i - n
|
||||
}
|
||||
}
|
||||
return -1
|
||||
return indexRabinKarp(s, sep)
|
||||
}
|
||||
|
||||
// Count counts the number of non-overlapping instances of sep in s.
|
||||
// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s.
|
||||
// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
|
||||
func Count(s, sep []byte) int {
|
||||
return countGeneric(s, sep)
|
||||
}
|
||||
|
||||
// primeRK is the prime base used in Rabin-Karp algorithm.
|
||||
const primeRK = 16777619
|
||||
|
||||
// hashStr returns the hash and the appropriate multiplicative
|
||||
// factor for use in Rabin-Karp algorithm.
|
||||
func hashStr(sep []byte) (uint32, uint32) {
|
||||
hash := uint32(0)
|
||||
for i := 0; i < len(sep); i++ {
|
||||
hash = hash*primeRK + uint32(sep[i])
|
||||
}
|
||||
var pow, sq uint32 = 1, primeRK
|
||||
for i := len(sep); i > 0; i >>= 1 {
|
||||
if i&1 != 0 {
|
||||
pow *= sq
|
||||
}
|
||||
sq *= sq
|
||||
}
|
||||
return hash, pow
|
||||
}
|
||||
|
|
|
@ -140,6 +140,9 @@ var indexTests = []BinOpTest{
|
|||
{"barfoobarfooyyyzzzyyyzzzyyyzzzyyyxxxzzzyyy", "x", 33},
|
||||
{"foofyfoobarfoobar", "y", 4},
|
||||
{"oooooooooooooooooooooo", "r", -1},
|
||||
// test fallback to Rabin-Karp.
|
||||
{"oxoxoxoxoxoxoxoxoxoxoxoy", "oy", 22},
|
||||
{"oxoxoxoxoxoxoxoxoxoxoxox", "oy", -1},
|
||||
}
|
||||
|
||||
var lastIndexTests = []BinOpTest{
|
||||
|
@ -741,6 +744,13 @@ var splittests = []SplitTest{
|
|||
func TestSplit(t *testing.T) {
|
||||
for _, tt := range splittests {
|
||||
a := SplitN([]byte(tt.s), []byte(tt.sep), tt.n)
|
||||
|
||||
// Appending to the results should not change future results.
|
||||
var x []byte
|
||||
for _, v := range a {
|
||||
x = append(v, 'z')
|
||||
}
|
||||
|
||||
result := sliceOfString(a)
|
||||
if !eq(result, tt.a) {
|
||||
t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a)
|
||||
|
@ -749,6 +759,11 @@ func TestSplit(t *testing.T) {
|
|||
if tt.n == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
|
||||
t.Errorf("last appended result was %s; want %s", x, want)
|
||||
}
|
||||
|
||||
s := Join(a, []byte(tt.sep))
|
||||
if string(s) != tt.s {
|
||||
t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s)
|
||||
|
@ -787,11 +802,23 @@ var splitaftertests = []SplitTest{
|
|||
func TestSplitAfter(t *testing.T) {
|
||||
for _, tt := range splitaftertests {
|
||||
a := SplitAfterN([]byte(tt.s), []byte(tt.sep), tt.n)
|
||||
|
||||
// Appending to the results should not change future results.
|
||||
var x []byte
|
||||
for _, v := range a {
|
||||
x = append(v, 'z')
|
||||
}
|
||||
|
||||
result := sliceOfString(a)
|
||||
if !eq(result, tt.a) {
|
||||
t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a)
|
||||
continue
|
||||
}
|
||||
|
||||
if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
|
||||
t.Errorf("last appended result was %s; want %s", x, want)
|
||||
}
|
||||
|
||||
s := Join(a, nil)
|
||||
if string(s) != tt.s {
|
||||
t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s)
|
||||
|
@ -826,12 +853,29 @@ var fieldstests = []FieldsTest{
|
|||
|
||||
func TestFields(t *testing.T) {
|
||||
for _, tt := range fieldstests {
|
||||
a := Fields([]byte(tt.s))
|
||||
b := []byte(tt.s)
|
||||
a := Fields(b)
|
||||
|
||||
// Appending to the results should not change future results.
|
||||
var x []byte
|
||||
for _, v := range a {
|
||||
x = append(v, 'z')
|
||||
}
|
||||
|
||||
result := sliceOfString(a)
|
||||
if !eq(result, tt.a) {
|
||||
t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a)
|
||||
continue
|
||||
}
|
||||
|
||||
if string(b) != tt.s {
|
||||
t.Errorf("slice changed to %s; want %s", string(b), tt.s)
|
||||
}
|
||||
if len(tt.a) > 0 {
|
||||
if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
|
||||
t.Errorf("last appended result was %s; want %s", x, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -852,11 +896,28 @@ func TestFieldsFunc(t *testing.T) {
|
|||
{"aXXbXXXcX", []string{"a", "b", "c"}},
|
||||
}
|
||||
for _, tt := range fieldsFuncTests {
|
||||
a := FieldsFunc([]byte(tt.s), pred)
|
||||
b := []byte(tt.s)
|
||||
a := FieldsFunc(b, pred)
|
||||
|
||||
// Appending to the results should not change future results.
|
||||
var x []byte
|
||||
for _, v := range a {
|
||||
x = append(v, 'z')
|
||||
}
|
||||
|
||||
result := sliceOfString(a)
|
||||
if !eq(result, tt.a) {
|
||||
t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a)
|
||||
}
|
||||
|
||||
if string(b) != tt.s {
|
||||
t.Errorf("slice changed to %s; want %s", b, tt.s)
|
||||
}
|
||||
if len(tt.a) > 0 {
|
||||
if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
|
||||
t.Errorf("last appended result was %s; want %s", x, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1507,19 +1568,58 @@ var makeFieldsInput = func() []byte {
|
|||
return x
|
||||
}
|
||||
|
||||
var fieldsInput = makeFieldsInput()
|
||||
var makeFieldsInputASCII = func() []byte {
|
||||
x := make([]byte, 1<<20)
|
||||
// Input is ~10% space, rest ASCII non-space.
|
||||
for i := range x {
|
||||
if rand.Intn(10) == 0 {
|
||||
x[i] = ' '
|
||||
} else {
|
||||
x[i] = 'x'
|
||||
}
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
var bytesdata = []struct {
|
||||
name string
|
||||
data []byte
|
||||
}{
|
||||
{"ASCII", makeFieldsInputASCII()},
|
||||
{"Mixed", makeFieldsInput()},
|
||||
}
|
||||
|
||||
func BenchmarkFields(b *testing.B) {
|
||||
b.SetBytes(int64(len(fieldsInput)))
|
||||
for i := 0; i < b.N; i++ {
|
||||
Fields(fieldsInput)
|
||||
for _, sd := range bytesdata {
|
||||
b.Run(sd.name, func(b *testing.B) {
|
||||
for j := 1 << 4; j <= 1<<20; j <<= 4 {
|
||||
b.Run(fmt.Sprintf("%d", j), func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(j))
|
||||
data := sd.data[:j]
|
||||
for i := 0; i < b.N; i++ {
|
||||
Fields(data)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFieldsFunc(b *testing.B) {
|
||||
b.SetBytes(int64(len(fieldsInput)))
|
||||
for i := 0; i < b.N; i++ {
|
||||
FieldsFunc(fieldsInput, unicode.IsSpace)
|
||||
for _, sd := range bytesdata {
|
||||
b.Run(sd.name, func(b *testing.B) {
|
||||
for j := 1 << 4; j <= 1<<20; j <<= 4 {
|
||||
b.Run(fmt.Sprintf("%d", j), func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(j))
|
||||
data := sd.data[:j]
|
||||
for i := 0; i < b.N; i++ {
|
||||
FieldsFunc(data, unicode.IsSpace)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1638,3 +1738,18 @@ func BenchmarkTrimASCII(b *testing.B) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkIndexPeriodic(b *testing.B) {
|
||||
key := []byte{1, 1}
|
||||
for _, skip := range [...]int{2, 4, 8, 16, 32, 64} {
|
||||
b.Run(fmt.Sprintf("IndexPeriodic%d", skip), func(b *testing.B) {
|
||||
buf := make([]byte, 1<<16)
|
||||
for i := 0; i < len(buf); i += skip {
|
||||
buf[i] = 1
|
||||
}
|
||||
for i := 0; i < b.N; i++ {
|
||||
Index(buf, key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,47 +0,0 @@
|
|||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
//
|
||||
// +build linux
|
||||
|
||||
package bytes_test
|
||||
|
||||
import (
|
||||
. "bytes"
|
||||
"syscall"
|
||||
"testing"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// This file tests the situation where memeq is checking
|
||||
// data very near to a page boundary. We want to make sure
|
||||
// equal does not read across the boundary and cause a page
|
||||
// fault where it shouldn't.
|
||||
|
||||
// This test runs only on linux. The code being tested is
|
||||
// not OS-specific, so it does not need to be tested on all
|
||||
// operating systems.
|
||||
|
||||
func TestEqualNearPageBoundary(t *testing.T) {
|
||||
pagesize := syscall.Getpagesize()
|
||||
b := make([]byte, 4*pagesize)
|
||||
i := pagesize
|
||||
for ; uintptr(unsafe.Pointer(&b[i]))%uintptr(pagesize) != 0; i++ {
|
||||
}
|
||||
syscall.Mprotect(b[i-pagesize:i], 0)
|
||||
syscall.Mprotect(b[i+pagesize:i+2*pagesize], 0)
|
||||
defer syscall.Mprotect(b[i-pagesize:i], syscall.PROT_READ|syscall.PROT_WRITE)
|
||||
defer syscall.Mprotect(b[i+pagesize:i+2*pagesize], syscall.PROT_READ|syscall.PROT_WRITE)
|
||||
|
||||
// both of these should fault
|
||||
//pagesize += int(b[i-1])
|
||||
//pagesize += int(b[i+pagesize])
|
||||
|
||||
for j := 0; j < pagesize; j++ {
|
||||
b[i+j] = 'A'
|
||||
}
|
||||
for j := 0; j <= pagesize; j++ {
|
||||
Equal(b[i:i+j], b[i+pagesize-j:i+pagesize])
|
||||
Equal(b[i+pagesize-j:i+pagesize], b[i:i+j])
|
||||
}
|
||||
}
|
|
@ -119,6 +119,32 @@ func ExampleContains() {
|
|||
// true
|
||||
}
|
||||
|
||||
func ExampleContainsAny() {
|
||||
fmt.Println(bytes.ContainsAny([]byte("I like seafood."), "fÄo!"))
|
||||
fmt.Println(bytes.ContainsAny([]byte("I like seafood."), "去是伟大的."))
|
||||
fmt.Println(bytes.ContainsAny([]byte("I like seafood."), ""))
|
||||
fmt.Println(bytes.ContainsAny([]byte(""), ""))
|
||||
// Output:
|
||||
// true
|
||||
// true
|
||||
// false
|
||||
// false
|
||||
}
|
||||
|
||||
func ExampleContainsRune() {
|
||||
fmt.Println(bytes.ContainsRune([]byte("I like seafood."), 'f'))
|
||||
fmt.Println(bytes.ContainsRune([]byte("I like seafood."), 'ö'))
|
||||
fmt.Println(bytes.ContainsRune([]byte("去是伟大的!"), '大'))
|
||||
fmt.Println(bytes.ContainsRune([]byte("去是伟大的!"), '!'))
|
||||
fmt.Println(bytes.ContainsRune([]byte(""), '@'))
|
||||
// Output:
|
||||
// true
|
||||
// false
|
||||
// true
|
||||
// true
|
||||
// false
|
||||
}
|
||||
|
||||
func ExampleCount() {
|
||||
fmt.Println(bytes.Count([]byte("cheese"), []byte("e")))
|
||||
fmt.Println(bytes.Count([]byte("five"), []byte(""))) // before & after each rune
|
||||
|
@ -127,6 +153,14 @@ func ExampleCount() {
|
|||
// 5
|
||||
}
|
||||
|
||||
func ExampleEqual() {
|
||||
fmt.Println(bytes.Equal([]byte("Go"), []byte("Go")))
|
||||
fmt.Println(bytes.Equal([]byte("Go"), []byte("C++")))
|
||||
// Output:
|
||||
// true
|
||||
// false
|
||||
}
|
||||
|
||||
func ExampleEqualFold() {
|
||||
fmt.Println(bytes.EqualFold([]byte("Go"), []byte("go")))
|
||||
// Output: true
|
||||
|
@ -162,6 +196,14 @@ func ExampleIndex() {
|
|||
// -1
|
||||
}
|
||||
|
||||
func ExampleIndexByte() {
|
||||
fmt.Println(bytes.IndexByte([]byte("chicken"), byte('k')))
|
||||
fmt.Println(bytes.IndexByte([]byte("chicken"), byte('g')))
|
||||
// Output:
|
||||
// 4
|
||||
// -1
|
||||
}
|
||||
|
||||
func ExampleIndexFunc() {
|
||||
f := func(c rune) bool {
|
||||
return unicode.Is(unicode.Han, c)
|
||||
|
@ -199,6 +241,36 @@ func ExampleLastIndex() {
|
|||
// -1
|
||||
}
|
||||
|
||||
func ExampleLastIndexAny() {
|
||||
fmt.Println(bytes.LastIndexAny([]byte("go gopher"), "MüQp"))
|
||||
fmt.Println(bytes.LastIndexAny([]byte("go 地鼠"), "地大"))
|
||||
fmt.Println(bytes.LastIndexAny([]byte("go gopher"), "z,!."))
|
||||
// Output:
|
||||
// 5
|
||||
// 3
|
||||
// -1
|
||||
}
|
||||
|
||||
func ExampleLastIndexByte() {
|
||||
fmt.Println(bytes.LastIndexByte([]byte("go gopher"), byte('g')))
|
||||
fmt.Println(bytes.LastIndexByte([]byte("go gopher"), byte('r')))
|
||||
fmt.Println(bytes.LastIndexByte([]byte("go gopher"), byte('z')))
|
||||
// Output:
|
||||
// 3
|
||||
// 8
|
||||
// -1
|
||||
}
|
||||
|
||||
func ExampleLastIndexFunc() {
|
||||
fmt.Println(bytes.LastIndexFunc([]byte("go gopher!"), unicode.IsLetter))
|
||||
fmt.Println(bytes.LastIndexFunc([]byte("go gopher!"), unicode.IsPunct))
|
||||
fmt.Println(bytes.LastIndexFunc([]byte("go gopher!"), unicode.IsNumber))
|
||||
// Output:
|
||||
// 8
|
||||
// 9
|
||||
// -1
|
||||
}
|
||||
|
||||
func ExampleJoin() {
|
||||
s := [][]byte{[]byte("foo"), []byte("bar"), []byte("baz")}
|
||||
fmt.Printf("%s", bytes.Join(s, []byte(", ")))
|
||||
|
@ -218,6 +290,23 @@ func ExampleReplace() {
|
|||
// moo moo moo
|
||||
}
|
||||
|
||||
func ExampleRunes() {
|
||||
rs := bytes.Runes([]byte("go gopher"))
|
||||
for _, r := range rs {
|
||||
fmt.Printf("%#U\n", r)
|
||||
}
|
||||
// Output:
|
||||
// U+0067 'g'
|
||||
// U+006F 'o'
|
||||
// U+0020 ' '
|
||||
// U+0067 'g'
|
||||
// U+006F 'o'
|
||||
// U+0070 'p'
|
||||
// U+0068 'h'
|
||||
// U+0065 'e'
|
||||
// U+0072 'r'
|
||||
}
|
||||
|
||||
func ExampleSplit() {
|
||||
fmt.Printf("%q\n", bytes.Split([]byte("a,b,c"), []byte(",")))
|
||||
fmt.Printf("%q\n", bytes.Split([]byte("a man a plan a canal panama"), []byte("a ")))
|
||||
|
@ -267,6 +356,18 @@ func ExampleTrim() {
|
|||
// Output: ["Achtung! Achtung"]
|
||||
}
|
||||
|
||||
func ExampleTrimFunc() {
|
||||
fmt.Println(string(bytes.TrimFunc([]byte("go-gopher!"), unicode.IsLetter)))
|
||||
fmt.Println(string(bytes.TrimFunc([]byte("\"go-gopher!\""), unicode.IsLetter)))
|
||||
fmt.Println(string(bytes.TrimFunc([]byte("go-gopher!"), unicode.IsPunct)))
|
||||
fmt.Println(string(bytes.TrimFunc([]byte("1234go-gopher!567"), unicode.IsNumber)))
|
||||
// Output:
|
||||
// -gopher!
|
||||
// "go-gopher!"
|
||||
// go-gopher
|
||||
// go-gopher!
|
||||
}
|
||||
|
||||
func ExampleMap() {
|
||||
rot13 := func(r rune) rune {
|
||||
switch {
|
||||
|
@ -281,11 +382,43 @@ func ExampleMap() {
|
|||
// Output: 'Gjnf oevyyvt naq gur fyvgul tbcure...
|
||||
}
|
||||
|
||||
func ExampleTrimLeft() {
|
||||
fmt.Print(string(bytes.TrimLeft([]byte("453gopher8257"), "0123456789")))
|
||||
// Output:
|
||||
// gopher8257
|
||||
}
|
||||
|
||||
func ExampleTrimLeftFunc() {
|
||||
fmt.Println(string(bytes.TrimLeftFunc([]byte("go-gopher"), unicode.IsLetter)))
|
||||
fmt.Println(string(bytes.TrimLeftFunc([]byte("go-gopher!"), unicode.IsPunct)))
|
||||
fmt.Println(string(bytes.TrimLeftFunc([]byte("1234go-gopher!567"), unicode.IsNumber)))
|
||||
// Output:
|
||||
// -gopher
|
||||
// go-gopher!
|
||||
// go-gopher!567
|
||||
}
|
||||
|
||||
func ExampleTrimSpace() {
|
||||
fmt.Printf("%s", bytes.TrimSpace([]byte(" \t\n a lone gopher \n\t\r\n")))
|
||||
// Output: a lone gopher
|
||||
}
|
||||
|
||||
func ExampleTrimRight() {
|
||||
fmt.Print(string(bytes.TrimRight([]byte("453gopher8257"), "0123456789")))
|
||||
// Output:
|
||||
// 453gopher
|
||||
}
|
||||
|
||||
func ExampleTrimRightFunc() {
|
||||
fmt.Println(string(bytes.TrimRightFunc([]byte("go-gopher"), unicode.IsLetter)))
|
||||
fmt.Println(string(bytes.TrimRightFunc([]byte("go-gopher!"), unicode.IsPunct)))
|
||||
fmt.Println(string(bytes.TrimRightFunc([]byte("1234go-gopher!567"), unicode.IsNumber)))
|
||||
// Output:
|
||||
// go-
|
||||
// go-gopher
|
||||
// 1234go-gopher!
|
||||
}
|
||||
|
||||
func ExampleToUpper() {
|
||||
fmt.Printf("%s", bytes.ToUpper([]byte("Gopher")))
|
||||
// Output: GOPHER
|
||||
|
@ -295,3 +428,11 @@ func ExampleToLower() {
|
|||
fmt.Printf("%s", bytes.ToLower([]byte("Gopher")))
|
||||
// Output: gopher
|
||||
}
|
||||
|
||||
func ExampleReader_Len() {
|
||||
fmt.Println(bytes.NewReader([]byte("Hi!")).Len())
|
||||
fmt.Println(bytes.NewReader([]byte("こんにちは!")).Len())
|
||||
// Output:
|
||||
// 3
|
||||
// 16
|
||||
}
|
||||
|
|
|
@ -35,6 +35,7 @@ func (r *Reader) Len() int {
|
|||
// to any other method.
|
||||
func (r *Reader) Size() int64 { return int64(len(r.s)) }
|
||||
|
||||
// Read implements the io.Reader interface.
|
||||
func (r *Reader) Read(b []byte) (n int, err error) {
|
||||
if r.i >= int64(len(r.s)) {
|
||||
return 0, io.EOF
|
||||
|
@ -45,6 +46,7 @@ func (r *Reader) Read(b []byte) (n int, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
// ReadAt implements the io.ReaderAt interface.
|
||||
func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) {
|
||||
// cannot modify state - see io.ReaderAt
|
||||
if off < 0 {
|
||||
|
@ -60,6 +62,7 @@ func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
// ReadByte implements the io.ByteReader interface.
|
||||
func (r *Reader) ReadByte() (byte, error) {
|
||||
r.prevRune = -1
|
||||
if r.i >= int64(len(r.s)) {
|
||||
|
@ -70,6 +73,7 @@ func (r *Reader) ReadByte() (byte, error) {
|
|||
return b, nil
|
||||
}
|
||||
|
||||
// UnreadByte complements ReadByte in implementing the io.ByteScanner interface.
|
||||
func (r *Reader) UnreadByte() error {
|
||||
r.prevRune = -1
|
||||
if r.i <= 0 {
|
||||
|
@ -79,6 +83,7 @@ func (r *Reader) UnreadByte() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// ReadRune implements the io.RuneReader interface.
|
||||
func (r *Reader) ReadRune() (ch rune, size int, err error) {
|
||||
if r.i >= int64(len(r.s)) {
|
||||
r.prevRune = -1
|
||||
|
@ -94,6 +99,7 @@ func (r *Reader) ReadRune() (ch rune, size int, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
// UnreadRune complements ReadRune in implementing the io.RuneScanner interface.
|
||||
func (r *Reader) UnreadRune() error {
|
||||
if r.prevRune < 0 {
|
||||
return errors.New("bytes.Reader.UnreadRune: previous operation was not ReadRune")
|
||||
|
|
|
@ -140,9 +140,9 @@ func TestReaderWriteTo(t *testing.T) {
|
|||
for i := 0; i < 30; i += 3 {
|
||||
var l int
|
||||
if i > 0 {
|
||||
l = len(data) / i
|
||||
l = len(testString) / i
|
||||
}
|
||||
s := data[:l]
|
||||
s := testString[:l]
|
||||
r := NewReader(testBytes[:l])
|
||||
var b Buffer
|
||||
n, err := r.WriteTo(&b)
|
||||
|
|
|
@ -0,0 +1,73 @@
|
|||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"cmd/internal/buildid"
|
||||
)
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintf(os.Stderr, "usage: go tool buildid [-w] file\n")
|
||||
flag.PrintDefaults()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
var wflag = flag.Bool("w", false, "write build ID")
|
||||
|
||||
func main() {
|
||||
log.SetPrefix("buildid: ")
|
||||
log.SetFlags(0)
|
||||
flag.Usage = usage
|
||||
flag.Parse()
|
||||
if flag.NArg() != 1 {
|
||||
usage()
|
||||
}
|
||||
|
||||
file := flag.Arg(0)
|
||||
id, err := buildid.ReadFile(file)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if !*wflag {
|
||||
fmt.Printf("%s\n", id)
|
||||
return
|
||||
}
|
||||
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
matches, hash, err := buildid.FindAndHash(f, id, 0)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
|
||||
tail := id
|
||||
if i := strings.LastIndex(id, "."); i >= 0 {
|
||||
tail = tail[i+1:]
|
||||
}
|
||||
if len(tail) != len(hash)*2 {
|
||||
log.Fatalf("%s: cannot find %d-byte hash in id %s", file, len(hash), id)
|
||||
}
|
||||
newID := id[:len(id)-len(tail)] + fmt.Sprintf("%x", hash)
|
||||
|
||||
f, err = os.OpenFile(file, os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := buildid.Rewrite(f, matches, newID); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
|
@ -0,0 +1,18 @@
|
|||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
/*
|
||||
Buildid displays or updates the build ID stored in a Go package or binary.
|
||||
|
||||
Usage:
|
||||
go tool buildid [-w] file
|
||||
|
||||
By default, buildid prints the build ID found in the named file.
|
||||
If the -w option is given, buildid rewrites the build ID found in
|
||||
the file to accurately record a content hash of the file.
|
||||
|
||||
This tool is only intended for use by the go command or
|
||||
other build systems.
|
||||
*/
|
||||
package main
|
|
@ -58,11 +58,14 @@ func (f *File) ParseGo(name string, src []byte) {
|
|||
// so we use ast1 to look for the doc comments on import "C"
|
||||
// and on exported functions, and we use ast2 for translating
|
||||
// and reprinting.
|
||||
// In cgo mode, we ignore ast2 and just apply edits directly
|
||||
// the text behind ast1. In godefs mode we modify and print ast2.
|
||||
ast1 := parse(name, src, parser.ParseComments)
|
||||
ast2 := parse(name, src, 0)
|
||||
|
||||
f.Package = ast1.Name.Name
|
||||
f.Name = make(map[string]*Name)
|
||||
f.NamePos = make(map[*Name]token.Pos)
|
||||
|
||||
// In ast1, find the import "C" line and get any extra C preamble.
|
||||
sawC := false
|
||||
|
@ -96,36 +99,53 @@ func (f *File) ParseGo(name string, src []byte) {
|
|||
}
|
||||
|
||||
// In ast2, strip the import "C" line.
|
||||
w := 0
|
||||
for _, decl := range ast2.Decls {
|
||||
d, ok := decl.(*ast.GenDecl)
|
||||
if !ok {
|
||||
ast2.Decls[w] = decl
|
||||
if *godefs {
|
||||
w := 0
|
||||
for _, decl := range ast2.Decls {
|
||||
d, ok := decl.(*ast.GenDecl)
|
||||
if !ok {
|
||||
ast2.Decls[w] = decl
|
||||
w++
|
||||
continue
|
||||
}
|
||||
ws := 0
|
||||
for _, spec := range d.Specs {
|
||||
s, ok := spec.(*ast.ImportSpec)
|
||||
if !ok || s.Path.Value != `"C"` {
|
||||
d.Specs[ws] = spec
|
||||
ws++
|
||||
}
|
||||
}
|
||||
if ws == 0 {
|
||||
continue
|
||||
}
|
||||
d.Specs = d.Specs[0:ws]
|
||||
ast2.Decls[w] = d
|
||||
w++
|
||||
continue
|
||||
}
|
||||
ws := 0
|
||||
for _, spec := range d.Specs {
|
||||
s, ok := spec.(*ast.ImportSpec)
|
||||
if !ok || s.Path.Value != `"C"` {
|
||||
d.Specs[ws] = spec
|
||||
ws++
|
||||
ast2.Decls = ast2.Decls[0:w]
|
||||
} else {
|
||||
for _, decl := range ast2.Decls {
|
||||
d, ok := decl.(*ast.GenDecl)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, spec := range d.Specs {
|
||||
if s, ok := spec.(*ast.ImportSpec); ok && s.Path.Value == `"C"` {
|
||||
// Replace "C" with _ "unsafe", to keep program valid.
|
||||
// (Deleting import statement or clause is not safe if it is followed
|
||||
// in the source by an explicit semicolon.)
|
||||
f.Edit.Replace(f.offset(s.Path.Pos()), f.offset(s.Path.End()), `_ "unsafe"`)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ws == 0 {
|
||||
continue
|
||||
}
|
||||
d.Specs = d.Specs[0:ws]
|
||||
ast2.Decls[w] = d
|
||||
w++
|
||||
}
|
||||
ast2.Decls = ast2.Decls[0:w]
|
||||
|
||||
// Accumulate pointers to uses of C.x.
|
||||
if f.Ref == nil {
|
||||
f.Ref = make([]*Ref, 0, 8)
|
||||
}
|
||||
f.walk(ast2, "prog", (*File).saveExprs)
|
||||
f.walk(ast2, ctxProg, (*File).saveExprs)
|
||||
|
||||
// Accumulate exported functions.
|
||||
// The comments are only on ast1 but we need to
|
||||
|
@ -133,8 +153,8 @@ func (f *File) ParseGo(name string, src []byte) {
|
|||
// The first walk fills in ExpFunc, and the
|
||||
// second walk changes the entries to
|
||||
// refer to ast2 instead.
|
||||
f.walk(ast1, "prog", (*File).saveExport)
|
||||
f.walk(ast2, "prog", (*File).saveExport2)
|
||||
f.walk(ast1, ctxProg, (*File).saveExport)
|
||||
f.walk(ast2, ctxProg, (*File).saveExport2)
|
||||
|
||||
f.Comments = ast1.Comments
|
||||
f.AST = ast2
|
||||
|
@ -143,9 +163,6 @@ func (f *File) ParseGo(name string, src []byte) {
|
|||
// Like ast.CommentGroup's Text method but preserves
|
||||
// leading blank lines, so that line numbers line up.
|
||||
func commentText(g *ast.CommentGroup) string {
|
||||
if g == nil {
|
||||
return ""
|
||||
}
|
||||
var pieces []string
|
||||
for _, com := range g.List {
|
||||
c := com.Text
|
||||
|
@ -165,7 +182,7 @@ func commentText(g *ast.CommentGroup) string {
|
|||
}
|
||||
|
||||
// Save various references we are going to need later.
|
||||
func (f *File) saveExprs(x interface{}, context string) {
|
||||
func (f *File) saveExprs(x interface{}, context astContext) {
|
||||
switch x := x.(type) {
|
||||
case *ast.Expr:
|
||||
switch (*x).(type) {
|
||||
|
@ -178,7 +195,7 @@ func (f *File) saveExprs(x interface{}, context string) {
|
|||
}
|
||||
|
||||
// Save references to C.xxx for later processing.
|
||||
func (f *File) saveRef(n *ast.Expr, context string) {
|
||||
func (f *File) saveRef(n *ast.Expr, context astContext) {
|
||||
sel := (*n).(*ast.SelectorExpr)
|
||||
// For now, assume that the only instance of capital C is when
|
||||
// used as the imported package identifier.
|
||||
|
@ -188,10 +205,10 @@ func (f *File) saveRef(n *ast.Expr, context string) {
|
|||
if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" {
|
||||
return
|
||||
}
|
||||
if context == "as2" {
|
||||
context = "expr"
|
||||
if context == ctxAssign2 {
|
||||
context = ctxExpr
|
||||
}
|
||||
if context == "embed-type" {
|
||||
if context == ctxEmbedType {
|
||||
error_(sel.Pos(), "cannot embed C type")
|
||||
}
|
||||
goname := sel.Sel.Name
|
||||
|
@ -212,6 +229,7 @@ func (f *File) saveRef(n *ast.Expr, context string) {
|
|||
Go: goname,
|
||||
}
|
||||
f.Name[goname] = name
|
||||
f.NamePos[name] = sel.Pos()
|
||||
}
|
||||
f.Ref = append(f.Ref, &Ref{
|
||||
Name: name,
|
||||
|
@ -221,7 +239,7 @@ func (f *File) saveRef(n *ast.Expr, context string) {
|
|||
}
|
||||
|
||||
// Save calls to C.xxx for later processing.
|
||||
func (f *File) saveCall(call *ast.CallExpr, context string) {
|
||||
func (f *File) saveCall(call *ast.CallExpr, context astContext) {
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return
|
||||
|
@ -229,12 +247,12 @@ func (f *File) saveCall(call *ast.CallExpr, context string) {
|
|||
if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" {
|
||||
return
|
||||
}
|
||||
c := &Call{Call: call, Deferred: context == "defer"}
|
||||
c := &Call{Call: call, Deferred: context == ctxDefer}
|
||||
f.Calls = append(f.Calls, c)
|
||||
}
|
||||
|
||||
// If a function should be exported add it to ExpFunc.
|
||||
func (f *File) saveExport(x interface{}, context string) {
|
||||
func (f *File) saveExport(x interface{}, context astContext) {
|
||||
n, ok := x.(*ast.FuncDecl)
|
||||
if !ok {
|
||||
return
|
||||
|
@ -274,7 +292,7 @@ func (f *File) saveExport(x interface{}, context string) {
|
|||
}
|
||||
|
||||
// Make f.ExpFunc[i] point at the Func from this AST instead of the other one.
|
||||
func (f *File) saveExport2(x interface{}, context string) {
|
||||
func (f *File) saveExport2(x interface{}, context astContext) {
|
||||
n, ok := x.(*ast.FuncDecl)
|
||||
if !ok {
|
||||
return
|
||||
|
@ -288,8 +306,30 @@ func (f *File) saveExport2(x interface{}, context string) {
|
|||
}
|
||||
}
|
||||
|
||||
type astContext int
|
||||
|
||||
const (
|
||||
ctxProg astContext = iota
|
||||
ctxEmbedType
|
||||
ctxType
|
||||
ctxStmt
|
||||
ctxExpr
|
||||
ctxField
|
||||
ctxParam
|
||||
ctxAssign2 // assignment of a single expression to two variables
|
||||
ctxSwitch
|
||||
ctxTypeSwitch
|
||||
ctxFile
|
||||
ctxDecl
|
||||
ctxSpec
|
||||
ctxDefer
|
||||
ctxCall // any function call other than ctxCall2
|
||||
ctxCall2 // function call whose result is assigned to two variables
|
||||
ctxSelector
|
||||
)
|
||||
|
||||
// walk walks the AST x, calling visit(f, x, context) for each node.
|
||||
func (f *File) walk(x interface{}, context string, visit func(*File, interface{}, string)) {
|
||||
func (f *File) walk(x interface{}, context astContext, visit func(*File, interface{}, astContext)) {
|
||||
visit(f, x, context)
|
||||
switch n := x.(type) {
|
||||
case *ast.Expr:
|
||||
|
@ -304,10 +344,10 @@ func (f *File) walk(x interface{}, context string, visit func(*File, interface{}
|
|||
|
||||
// These are ordered and grouped to match ../../go/ast/ast.go
|
||||
case *ast.Field:
|
||||
if len(n.Names) == 0 && context == "field" {
|
||||
f.walk(&n.Type, "embed-type", visit)
|
||||
if len(n.Names) == 0 && context == ctxField {
|
||||
f.walk(&n.Type, ctxEmbedType, visit)
|
||||
} else {
|
||||
f.walk(&n.Type, "type", visit)
|
||||
f.walk(&n.Type, ctxType, visit)
|
||||
}
|
||||
case *ast.FieldList:
|
||||
for _, field := range n.List {
|
||||
|
@ -318,163 +358,163 @@ func (f *File) walk(x interface{}, context string, visit func(*File, interface{}
|
|||
case *ast.Ellipsis:
|
||||
case *ast.BasicLit:
|
||||
case *ast.FuncLit:
|
||||
f.walk(n.Type, "type", visit)
|
||||
f.walk(n.Body, "stmt", visit)
|
||||
f.walk(n.Type, ctxType, visit)
|
||||
f.walk(n.Body, ctxStmt, visit)
|
||||
case *ast.CompositeLit:
|
||||
f.walk(&n.Type, "type", visit)
|
||||
f.walk(n.Elts, "expr", visit)
|
||||
f.walk(&n.Type, ctxType, visit)
|
||||
f.walk(n.Elts, ctxExpr, visit)
|
||||
case *ast.ParenExpr:
|
||||
f.walk(&n.X, context, visit)
|
||||
case *ast.SelectorExpr:
|
||||
f.walk(&n.X, "selector", visit)
|
||||
f.walk(&n.X, ctxSelector, visit)
|
||||
case *ast.IndexExpr:
|
||||
f.walk(&n.X, "expr", visit)
|
||||
f.walk(&n.Index, "expr", visit)
|
||||
f.walk(&n.X, ctxExpr, visit)
|
||||
f.walk(&n.Index, ctxExpr, visit)
|
||||
case *ast.SliceExpr:
|
||||
f.walk(&n.X, "expr", visit)
|
||||
f.walk(&n.X, ctxExpr, visit)
|
||||
if n.Low != nil {
|
||||
f.walk(&n.Low, "expr", visit)
|
||||
f.walk(&n.Low, ctxExpr, visit)
|
||||
}
|
||||
if n.High != nil {
|
||||
f.walk(&n.High, "expr", visit)
|
||||
f.walk(&n.High, ctxExpr, visit)
|
||||
}
|
||||
if n.Max != nil {
|
||||
f.walk(&n.Max, "expr", visit)
|
||||
f.walk(&n.Max, ctxExpr, visit)
|
||||
}
|
||||
case *ast.TypeAssertExpr:
|
||||
f.walk(&n.X, "expr", visit)
|
||||
f.walk(&n.Type, "type", visit)
|
||||
f.walk(&n.X, ctxExpr, visit)
|
||||
f.walk(&n.Type, ctxType, visit)
|
||||
case *ast.CallExpr:
|
||||
if context == "as2" {
|
||||
f.walk(&n.Fun, "call2", visit)
|
||||
if context == ctxAssign2 {
|
||||
f.walk(&n.Fun, ctxCall2, visit)
|
||||
} else {
|
||||
f.walk(&n.Fun, "call", visit)
|
||||
f.walk(&n.Fun, ctxCall, visit)
|
||||
}
|
||||
f.walk(n.Args, "expr", visit)
|
||||
f.walk(n.Args, ctxExpr, visit)
|
||||
case *ast.StarExpr:
|
||||
f.walk(&n.X, context, visit)
|
||||
case *ast.UnaryExpr:
|
||||
f.walk(&n.X, "expr", visit)
|
||||
f.walk(&n.X, ctxExpr, visit)
|
||||
case *ast.BinaryExpr:
|
||||
f.walk(&n.X, "expr", visit)
|
||||
f.walk(&n.Y, "expr", visit)
|
||||
f.walk(&n.X, ctxExpr, visit)
|
||||
f.walk(&n.Y, ctxExpr, visit)
|
||||
case *ast.KeyValueExpr:
|
||||
f.walk(&n.Key, "expr", visit)
|
||||
f.walk(&n.Value, "expr", visit)
|
||||
f.walk(&n.Key, ctxExpr, visit)
|
||||
f.walk(&n.Value, ctxExpr, visit)
|
||||
|
||||
case *ast.ArrayType:
|
||||
f.walk(&n.Len, "expr", visit)
|
||||
f.walk(&n.Elt, "type", visit)
|
||||
f.walk(&n.Len, ctxExpr, visit)
|
||||
f.walk(&n.Elt, ctxType, visit)
|
||||
case *ast.StructType:
|
||||
f.walk(n.Fields, "field", visit)
|
||||
f.walk(n.Fields, ctxField, visit)
|
||||
case *ast.FuncType:
|
||||
f.walk(n.Params, "param", visit)
|
||||
f.walk(n.Params, ctxParam, visit)
|
||||
if n.Results != nil {
|
||||
f.walk(n.Results, "param", visit)
|
||||
f.walk(n.Results, ctxParam, visit)
|
||||
}
|
||||
case *ast.InterfaceType:
|
||||
f.walk(n.Methods, "field", visit)
|
||||
f.walk(n.Methods, ctxField, visit)
|
||||
case *ast.MapType:
|
||||
f.walk(&n.Key, "type", visit)
|
||||
f.walk(&n.Value, "type", visit)
|
||||
f.walk(&n.Key, ctxType, visit)
|
||||
f.walk(&n.Value, ctxType, visit)
|
||||
case *ast.ChanType:
|
||||
f.walk(&n.Value, "type", visit)
|
||||
f.walk(&n.Value, ctxType, visit)
|
||||
|
||||
case *ast.BadStmt:
|
||||
case *ast.DeclStmt:
|
||||
f.walk(n.Decl, "decl", visit)
|
||||
f.walk(n.Decl, ctxDecl, visit)
|
||||
case *ast.EmptyStmt:
|
||||
case *ast.LabeledStmt:
|
||||
f.walk(n.Stmt, "stmt", visit)
|
||||
f.walk(n.Stmt, ctxStmt, visit)
|
||||
case *ast.ExprStmt:
|
||||
f.walk(&n.X, "expr", visit)
|
||||
f.walk(&n.X, ctxExpr, visit)
|
||||
case *ast.SendStmt:
|
||||
f.walk(&n.Chan, "expr", visit)
|
||||
f.walk(&n.Value, "expr", visit)
|
||||
f.walk(&n.Chan, ctxExpr, visit)
|
||||
f.walk(&n.Value, ctxExpr, visit)
|
||||
case *ast.IncDecStmt:
|
||||
f.walk(&n.X, "expr", visit)
|
||||
f.walk(&n.X, ctxExpr, visit)
|
||||
case *ast.AssignStmt:
|
||||
f.walk(n.Lhs, "expr", visit)
|
||||
f.walk(n.Lhs, ctxExpr, visit)
|
||||
if len(n.Lhs) == 2 && len(n.Rhs) == 1 {
|
||||
f.walk(n.Rhs, "as2", visit)
|
||||
f.walk(n.Rhs, ctxAssign2, visit)
|
||||
} else {
|
||||
f.walk(n.Rhs, "expr", visit)
|
||||
f.walk(n.Rhs, ctxExpr, visit)
|
||||
}
|
||||
case *ast.GoStmt:
|
||||
f.walk(n.Call, "expr", visit)
|
||||
f.walk(n.Call, ctxExpr, visit)
|
||||
case *ast.DeferStmt:
|
||||
f.walk(n.Call, "defer", visit)
|
||||
f.walk(n.Call, ctxDefer, visit)
|
||||
case *ast.ReturnStmt:
|
||||
f.walk(n.Results, "expr", visit)
|
||||
f.walk(n.Results, ctxExpr, visit)
|
||||
case *ast.BranchStmt:
|
||||
case *ast.BlockStmt:
|
||||
f.walk(n.List, context, visit)
|
||||
case *ast.IfStmt:
|
||||
f.walk(n.Init, "stmt", visit)
|
||||
f.walk(&n.Cond, "expr", visit)
|
||||
f.walk(n.Body, "stmt", visit)
|
||||
f.walk(n.Else, "stmt", visit)
|
||||
f.walk(n.Init, ctxStmt, visit)
|
||||
f.walk(&n.Cond, ctxExpr, visit)
|
||||
f.walk(n.Body, ctxStmt, visit)
|
||||
f.walk(n.Else, ctxStmt, visit)
|
||||
case *ast.CaseClause:
|
||||
if context == "typeswitch" {
|
||||
context = "type"
|
||||
if context == ctxTypeSwitch {
|
||||
context = ctxType
|
||||
} else {
|
||||
context = "expr"
|
||||
context = ctxExpr
|
||||
}
|
||||
f.walk(n.List, context, visit)
|
||||
f.walk(n.Body, "stmt", visit)
|
||||
f.walk(n.Body, ctxStmt, visit)
|
||||
case *ast.SwitchStmt:
|
||||
f.walk(n.Init, "stmt", visit)
|
||||
f.walk(&n.Tag, "expr", visit)
|
||||
f.walk(n.Body, "switch", visit)
|
||||
f.walk(n.Init, ctxStmt, visit)
|
||||
f.walk(&n.Tag, ctxExpr, visit)
|
||||
f.walk(n.Body, ctxSwitch, visit)
|
||||
case *ast.TypeSwitchStmt:
|
||||
f.walk(n.Init, "stmt", visit)
|
||||
f.walk(n.Assign, "stmt", visit)
|
||||
f.walk(n.Body, "typeswitch", visit)
|
||||
f.walk(n.Init, ctxStmt, visit)
|
||||
f.walk(n.Assign, ctxStmt, visit)
|
||||
f.walk(n.Body, ctxTypeSwitch, visit)
|
||||
case *ast.CommClause:
|
||||
f.walk(n.Comm, "stmt", visit)
|
||||
f.walk(n.Body, "stmt", visit)
|
||||
f.walk(n.Comm, ctxStmt, visit)
|
||||
f.walk(n.Body, ctxStmt, visit)
|
||||
case *ast.SelectStmt:
|
||||
f.walk(n.Body, "stmt", visit)
|
||||
f.walk(n.Body, ctxStmt, visit)
|
||||
case *ast.ForStmt:
|
||||
f.walk(n.Init, "stmt", visit)
|
||||
f.walk(&n.Cond, "expr", visit)
|
||||
f.walk(n.Post, "stmt", visit)
|
||||
f.walk(n.Body, "stmt", visit)
|
||||
f.walk(n.Init, ctxStmt, visit)
|
||||
f.walk(&n.Cond, ctxExpr, visit)
|
||||
f.walk(n.Post, ctxStmt, visit)
|
||||
f.walk(n.Body, ctxStmt, visit)
|
||||
case *ast.RangeStmt:
|
||||
f.walk(&n.Key, "expr", visit)
|
||||
f.walk(&n.Value, "expr", visit)
|
||||
f.walk(&n.X, "expr", visit)
|
||||
f.walk(n.Body, "stmt", visit)
|
||||
f.walk(&n.Key, ctxExpr, visit)
|
||||
f.walk(&n.Value, ctxExpr, visit)
|
||||
f.walk(&n.X, ctxExpr, visit)
|
||||
f.walk(n.Body, ctxStmt, visit)
|
||||
|
||||
case *ast.ImportSpec:
|
||||
case *ast.ValueSpec:
|
||||
f.walk(&n.Type, "type", visit)
|
||||
f.walk(&n.Type, ctxType, visit)
|
||||
if len(n.Names) == 2 && len(n.Values) == 1 {
|
||||
f.walk(&n.Values[0], "as2", visit)
|
||||
f.walk(&n.Values[0], ctxAssign2, visit)
|
||||
} else {
|
||||
f.walk(n.Values, "expr", visit)
|
||||
f.walk(n.Values, ctxExpr, visit)
|
||||
}
|
||||
case *ast.TypeSpec:
|
||||
f.walk(&n.Type, "type", visit)
|
||||
f.walk(&n.Type, ctxType, visit)
|
||||
|
||||
case *ast.BadDecl:
|
||||
case *ast.GenDecl:
|
||||
f.walk(n.Specs, "spec", visit)
|
||||
f.walk(n.Specs, ctxSpec, visit)
|
||||
case *ast.FuncDecl:
|
||||
if n.Recv != nil {
|
||||
f.walk(n.Recv, "param", visit)
|
||||
f.walk(n.Recv, ctxParam, visit)
|
||||
}
|
||||
f.walk(n.Type, "type", visit)
|
||||
f.walk(n.Type, ctxType, visit)
|
||||
if n.Body != nil {
|
||||
f.walk(n.Body, "stmt", visit)
|
||||
f.walk(n.Body, ctxStmt, visit)
|
||||
}
|
||||
|
||||
case *ast.File:
|
||||
f.walk(n.Decls, "decl", visit)
|
||||
f.walk(n.Decls, ctxDecl, visit)
|
||||
|
||||
case *ast.Package:
|
||||
for _, file := range n.Files {
|
||||
f.walk(file, "file", visit)
|
||||
f.walk(file, ctxFile, visit)
|
||||
}
|
||||
|
||||
case []ast.Decl:
|
||||
|
|
|
@ -102,11 +102,13 @@ the use of cgo, and to 0 to disable it. The go tool will set the
|
|||
build constraint "cgo" if cgo is enabled.
|
||||
|
||||
When cross-compiling, you must specify a C cross-compiler for cgo to
|
||||
use. You can do this by setting the CC_FOR_TARGET environment
|
||||
variable when building the toolchain using make.bash, or by setting
|
||||
the CC environment variable any time you run the go tool. The
|
||||
CXX_FOR_TARGET and CXX environment variables work in a similar way for
|
||||
C++ code.
|
||||
use. You can do this by setting the generic CC_FOR_TARGET or the
|
||||
more specific CC_FOR_${GOOS}_${GOARCH} (for example, CC_FOR_linux_arm)
|
||||
environment variable when building the toolchain using make.bash,
|
||||
or you can set the CC environment variable any time you run the go tool.
|
||||
|
||||
The CXX_FOR_TARGET, CXX_FOR_${GOOS}_${GOARCH}, and CXX
|
||||
environment variables work in a similar way for C++ code.
|
||||
|
||||
Go references to C
|
||||
|
||||
|
@ -126,12 +128,29 @@ C.complexfloat (complex float), and C.complexdouble (complex double).
|
|||
The C type void* is represented by Go's unsafe.Pointer.
|
||||
The C types __int128_t and __uint128_t are represented by [16]byte.
|
||||
|
||||
A few special C types which would normally be represented by a pointer
|
||||
type in Go are instead represented by a uintptr. See the Special
|
||||
cases section below.
|
||||
|
||||
To access a struct, union, or enum type directly, prefix it with
|
||||
struct_, union_, or enum_, as in C.struct_stat.
|
||||
|
||||
The size of any C type T is available as C.sizeof_T, as in
|
||||
C.sizeof_struct_stat.
|
||||
|
||||
A C function may be declared in the Go file with a parameter type of
|
||||
the special name _GoString_. This function may be called with an
|
||||
ordinary Go string value. The string length, and a pointer to the
|
||||
string contents, may be accessed by calling the C functions
|
||||
|
||||
size_t _GoStringLen(_GoString_ s);
|
||||
const char *_GoStringPtr(_GoString_ s);
|
||||
|
||||
These functions are only available in the preamble, not in other C
|
||||
files. The C code must not modify the contents of the pointer returned
|
||||
by _GoStringPtr. Note that the string contents may not have a trailing
|
||||
NUL byte.
|
||||
|
||||
As Go doesn't have support for C's union type in the general case,
|
||||
C's union types are represented as a Go byte array with the same length.
|
||||
|
||||
|
@ -241,7 +260,16 @@ They will be available in the C code as:
|
|||
found in the _cgo_export.h generated header, after any preambles
|
||||
copied from the cgo input files. Functions with multiple
|
||||
return values are mapped to functions returning a struct.
|
||||
|
||||
Not all Go types can be mapped to C types in a useful way.
|
||||
Go struct types are not supported; use a C struct type.
|
||||
Go array types are not supported; use a C pointer.
|
||||
|
||||
Go functions that take arguments of type string may be called with the
|
||||
C type _GoString_, described above. The _GoString_ type will be
|
||||
automatically defined in the preamble. Note that there is no way for C
|
||||
code to create a value of this type; this is only useful for passing
|
||||
string values from Go to C and back to Go.
|
||||
|
||||
Using //export in a file places a restriction on the preamble:
|
||||
since it is copied into two different C output files, it must not
|
||||
|
@ -264,6 +292,14 @@ pointer is a Go pointer or a C pointer is a dynamic property
|
|||
determined by how the memory was allocated; it has nothing to do with
|
||||
the type of the pointer.
|
||||
|
||||
Note that values of some Go types, other than the type's zero value,
|
||||
always include Go pointers. This is true of string, slice, interface,
|
||||
channel, map, and function types. A pointer type may hold a Go pointer
|
||||
or a C pointer. Array and struct types may or may not include Go
|
||||
pointers, depending on the element types. All the discussion below
|
||||
about Go pointers applies not just to pointer types, but also to other
|
||||
types that include Go pointers.
|
||||
|
||||
Go code may pass a Go pointer to C provided the Go memory to which it
|
||||
points does not contain any Go pointers. The C code must preserve
|
||||
this property: it must not store any Go pointers in Go memory, even
|
||||
|
@ -274,14 +310,17 @@ the Go memory in question is the entire array or the entire backing
|
|||
array of the slice.
|
||||
|
||||
C code may not keep a copy of a Go pointer after the call returns.
|
||||
This includes the _GoString_ type, which, as noted above, includes a
|
||||
Go pointer; _GoString_ values may not be retained by C code.
|
||||
|
||||
A Go function called by C code may not return a Go pointer. A Go
|
||||
function called by C code may take C pointers as arguments, and it may
|
||||
store non-pointer or C pointer data through those pointers, but it may
|
||||
not store a Go pointer in memory pointed to by a C pointer. A Go
|
||||
function called by C code may take a Go pointer as an argument, but it
|
||||
must preserve the property that the Go memory to which it points does
|
||||
not contain any Go pointers.
|
||||
A Go function called by C code may not return a Go pointer (which
|
||||
implies that it may not return a string, slice, channel, and so
|
||||
forth). A Go function called by C code may take C pointers as
|
||||
arguments, and it may store non-pointer or C pointer data through
|
||||
those pointers, but it may not store a Go pointer in memory pointed to
|
||||
by a C pointer. A Go function called by C code may take a Go pointer
|
||||
as an argument, but it must preserve the property that the Go memory
|
||||
to which it points does not contain any Go pointers.
|
||||
|
||||
Go code may not store a Go pointer in C memory. C code may store Go
|
||||
pointers in C memory, subject to the rule above: it must stop storing
|
||||
|
@ -299,6 +338,84 @@ and of course there is nothing stopping the C code from doing anything
|
|||
it likes. However, programs that break these rules are likely to fail
|
||||
in unexpected and unpredictable ways.
|
||||
|
||||
Special cases
|
||||
|
||||
A few special C types which would normally be represented by a pointer
|
||||
type in Go are instead represented by a uintptr. Those types are
|
||||
the CF*Ref types from the CoreFoundation library on Darwin, including:
|
||||
|
||||
CFAllocatorRef
|
||||
CFArrayRef
|
||||
CFAttributedStringRef
|
||||
CFBagRef
|
||||
CFBinaryHeapRef
|
||||
CFBitVectorRef
|
||||
CFBooleanRef
|
||||
CFBundleRef
|
||||
CFCalendarRef
|
||||
CFCharacterSetRef
|
||||
CFDataRef
|
||||
CFDateFormatterRef
|
||||
CFDateRef
|
||||
CFDictionaryRef
|
||||
CFErrorRef
|
||||
CFFileDescriptorRef
|
||||
CFFileSecurityRef
|
||||
CFLocaleRef
|
||||
CFMachPortRef
|
||||
CFMessagePortRef
|
||||
CFMutableArrayRef
|
||||
CFMutableAttributedStringRef
|
||||
CFMutableBagRef
|
||||
CFMutableBitVectorRef
|
||||
CFMutableCharacterSetRef
|
||||
CFMutableDataRef
|
||||
CFMutableDictionaryRef
|
||||
CFMutableSetRef
|
||||
CFMutableStringRef
|
||||
CFNotificationCenterRef
|
||||
CFNullRef
|
||||
CFNumberFormatterRef
|
||||
CFNumberRef
|
||||
CFPlugInInstanceRef
|
||||
CFPlugInRef
|
||||
CFPropertyListRef
|
||||
CFReadStreamRef
|
||||
CFRunLoopObserverRef
|
||||
CFRunLoopRef
|
||||
CFRunLoopSourceRef
|
||||
CFRunLoopTimerRef
|
||||
CFSetRef
|
||||
CFSocketRef
|
||||
CFStringRef
|
||||
CFStringTokenizerRef
|
||||
CFTimeZoneRef
|
||||
CFTreeRef
|
||||
CFTypeRef
|
||||
CFURLCreateFromFSRef
|
||||
CFURLEnumeratorRef
|
||||
CFURLGetFSRef
|
||||
CFURLRef
|
||||
CFUUIDRef
|
||||
CFUserNotificationRef
|
||||
CFWriteStreamRef
|
||||
CFXMLNodeRef
|
||||
CFXMLParserRef
|
||||
CFXMLTreeRef
|
||||
|
||||
These types are uintptr on the Go side because they would otherwise
|
||||
confuse the Go garbage collector; they are sometimes not really
|
||||
pointers but data structures encoded in a pointer type. All operations
|
||||
on these types must happen in C. The proper constant to initialize an
|
||||
empty such reference is 0, not nil.
|
||||
|
||||
This special case was introduced in Go 1.10. For auto-updating code
|
||||
from Go 1.9 and earlier, use the cftype rewrite in the Go fix tool:
|
||||
|
||||
go tool fix -r cftype <pkg>
|
||||
|
||||
It will replace nil with 0 in the appropriate places.
|
||||
|
||||
Using cgo directly
|
||||
|
||||
Usage:
|
||||
|
@ -312,32 +429,35 @@ invoking the C compiler to compile the C parts of the package.
|
|||
|
||||
The following options are available when running cgo directly:
|
||||
|
||||
-V
|
||||
Print cgo version and exit.
|
||||
-debug-define
|
||||
Debugging option. Print #defines.
|
||||
-debug-gcc
|
||||
Debugging option. Trace C compiler execution and output.
|
||||
-dynimport file
|
||||
Write list of symbols imported by file. Write to
|
||||
-dynout argument or to standard output. Used by go
|
||||
build when building a cgo package.
|
||||
-dynlinker
|
||||
Write dynamic linker as part of -dynimport output.
|
||||
-dynout file
|
||||
Write -dynimport output to file.
|
||||
-dynpackage package
|
||||
Set Go package for -dynimport output.
|
||||
-dynlinker
|
||||
Write dynamic linker as part of -dynimport output.
|
||||
-godefs
|
||||
Write out input file in Go syntax replacing C package
|
||||
names with real values. Used to generate files in the
|
||||
syscall package when bootstrapping a new target.
|
||||
-srcdir directory
|
||||
Find the Go input files, listed on the command line,
|
||||
in directory.
|
||||
-objdir directory
|
||||
Put all generated files in directory.
|
||||
-importpath string
|
||||
The import path for the Go package. Optional; used for
|
||||
nicer comments in the generated files.
|
||||
-exportheader file
|
||||
If there are any exported functions, write the
|
||||
generated export declarations to file.
|
||||
C code can #include this to see the declarations.
|
||||
-importpath string
|
||||
The import path for the Go package. Optional; used for
|
||||
nicer comments in the generated files.
|
||||
-import_runtime_cgo
|
||||
If set (which it is by default) import runtime/cgo in
|
||||
generated output.
|
||||
-import_syscall
|
||||
If set (which it is by default) import syscall in
|
||||
generated output.
|
||||
-gccgo
|
||||
Generate output for the gccgo compiler rather than the
|
||||
gc compiler.
|
||||
|
@ -345,16 +465,13 @@ The following options are available when running cgo directly:
|
|||
The -fgo-prefix option to be used with gccgo.
|
||||
-gccgopkgpath path
|
||||
The -fgo-pkgpath option to be used with gccgo.
|
||||
-import_runtime_cgo
|
||||
If set (which it is by default) import runtime/cgo in
|
||||
generated output.
|
||||
-import_syscall
|
||||
If set (which it is by default) import syscall in
|
||||
generated output.
|
||||
-debug-define
|
||||
Debugging option. Print #defines.
|
||||
-debug-gcc
|
||||
Debugging option. Trace C compiler execution and output.
|
||||
-godefs
|
||||
Write out input file in Go syntax replacing C package
|
||||
names with real values. Used to generate files in the
|
||||
syscall package when bootstrapping a new target.
|
||||
-objdir directory
|
||||
Put all generated files in directory.
|
||||
-srcdir directory
|
||||
*/
|
||||
package main
|
||||
|
||||
|
@ -403,21 +520,19 @@ about simple #defines for constants and the like. These are recorded
|
|||
for later use.
|
||||
|
||||
Next, cgo needs to identify the kinds for each identifier. For the
|
||||
identifiers C.foo and C.bar, cgo generates this C program:
|
||||
identifiers C.foo, cgo generates this C program:
|
||||
|
||||
<preamble>
|
||||
#line 1 "not-declared"
|
||||
void __cgo_f_xxx_1(void) { __typeof__(foo) *__cgo_undefined__; }
|
||||
void __cgo_f_1_1(void) { __typeof__(foo) *__cgo_undefined__1; }
|
||||
#line 1 "not-type"
|
||||
void __cgo_f_xxx_2(void) { foo *__cgo_undefined__; }
|
||||
#line 1 "not-const"
|
||||
void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (foo)*1 }; }
|
||||
#line 2 "not-declared"
|
||||
void __cgo_f_xxx_1(void) { __typeof__(bar) *__cgo_undefined__; }
|
||||
#line 2 "not-type"
|
||||
void __cgo_f_xxx_2(void) { bar *__cgo_undefined__; }
|
||||
#line 2 "not-const"
|
||||
void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (bar)*1 }; }
|
||||
void __cgo_f_1_2(void) { foo *__cgo_undefined__2; }
|
||||
#line 1 "not-int-const"
|
||||
void __cgo_f_1_3(void) { enum { __cgo_undefined__3 = (foo)*1 }; }
|
||||
#line 1 "not-num-const"
|
||||
void __cgo_f_1_4(void) { static const double __cgo_undefined__4 = (foo); }
|
||||
#line 1 "not-str-lit"
|
||||
void __cgo_f_1_5(void) { static const char __cgo_undefined__5[] = (foo); }
|
||||
|
||||
This program will not compile, but cgo can use the presence or absence
|
||||
of an error message on a given line to deduce the information it
|
||||
|
@ -427,45 +542,72 @@ errors that might stop parsing early.
|
|||
|
||||
An error on not-declared:1 indicates that foo is undeclared.
|
||||
An error on not-type:1 indicates that foo is not a type (if declared at all, it is an identifier).
|
||||
An error on not-const:1 indicates that foo is not an integer constant.
|
||||
An error on not-int-const:1 indicates that foo is not an integer constant.
|
||||
An error on not-num-const:1 indicates that foo is not a number constant.
|
||||
An error on not-str-lit:1 indicates that foo is not a string literal.
|
||||
An error on not-signed-int-const:1 indicates that foo is not a signed integer constant.
|
||||
|
||||
The line number specifies the name involved. In the example, 1 is foo and 2 is bar.
|
||||
The line number specifies the name involved. In the example, 1 is foo.
|
||||
|
||||
Next, cgo must learn the details of each type, variable, function, or
|
||||
constant. It can do this by reading object files. If cgo has decided
|
||||
that t1 is a type, v2 and v3 are variables or functions, and c4, c5,
|
||||
and c6 are constants, it generates:
|
||||
that t1 is a type, v2 and v3 are variables or functions, and i4, i5
|
||||
are integer constants, u6 is an unsigned integer constant, and f7 and f8
|
||||
are float constants, and s9 and s10 are string constants, it generates:
|
||||
|
||||
<preamble>
|
||||
__typeof__(t1) *__cgo__1;
|
||||
__typeof__(v2) *__cgo__2;
|
||||
__typeof__(v3) *__cgo__3;
|
||||
__typeof__(c4) *__cgo__4;
|
||||
enum { __cgo_enum__4 = c4 };
|
||||
__typeof__(c5) *__cgo__5;
|
||||
enum { __cgo_enum__5 = c5 };
|
||||
__typeof__(c6) *__cgo__6;
|
||||
enum { __cgo_enum__6 = c6 };
|
||||
__typeof__(i4) *__cgo__4;
|
||||
enum { __cgo_enum__4 = i4 };
|
||||
__typeof__(i5) *__cgo__5;
|
||||
enum { __cgo_enum__5 = i5 };
|
||||
__typeof__(u6) *__cgo__6;
|
||||
enum { __cgo_enum__6 = u6 };
|
||||
__typeof__(f7) *__cgo__7;
|
||||
__typeof__(f8) *__cgo__8;
|
||||
__typeof__(s9) *__cgo__9;
|
||||
__typeof__(s10) *__cgo__10;
|
||||
|
||||
long long __cgo_debug_data[] = {
|
||||
long long __cgodebug_ints[] = {
|
||||
0, // t1
|
||||
0, // v2
|
||||
0, // v3
|
||||
c4,
|
||||
c5,
|
||||
c6,
|
||||
i4,
|
||||
i5,
|
||||
u6,
|
||||
0, // f7
|
||||
0, // f8
|
||||
0, // s9
|
||||
0, // s10
|
||||
1
|
||||
};
|
||||
|
||||
double __cgodebug_floats[] = {
|
||||
0, // t1
|
||||
0, // v2
|
||||
0, // v3
|
||||
0, // i4
|
||||
0, // i5
|
||||
0, // u6
|
||||
f7,
|
||||
f8,
|
||||
0, // s9
|
||||
0, // s10
|
||||
1
|
||||
};
|
||||
|
||||
const char __cgodebug_str__9[] = s9;
|
||||
const unsigned long long __cgodebug_strlen__9 = sizeof(s9)-1;
|
||||
const char __cgodebug_str__10[] = s10;
|
||||
const unsigned long long __cgodebug_strlen__10 = sizeof(s10)-1;
|
||||
|
||||
and again invokes the system C compiler, to produce an object file
|
||||
containing debug information. Cgo parses the DWARF debug information
|
||||
for __cgo__N to learn the type of each identifier. (The types also
|
||||
distinguish functions from global variables.) If using a standard gcc,
|
||||
cgo can parse the DWARF debug information for the __cgo_enum__N to
|
||||
learn the identifier's value. The LLVM-based gcc on OS X emits
|
||||
incomplete DWARF information for enums; in that case cgo reads the
|
||||
constant values from the __cgo_debug_data from the object file's data
|
||||
segment.
|
||||
distinguish functions from global variables.) Cgo reads the constant
|
||||
values from the __cgodebug_* from the object file's data segment.
|
||||
|
||||
At this point cgo knows the meaning of each C.xxx well enough to start
|
||||
the translation process.
|
||||
|
@ -550,9 +692,12 @@ _cgo_main.c:
|
|||
|
||||
int main() { return 0; }
|
||||
void crosscall2(void(*fn)(void*, int, uintptr_t), void *a, int c, uintptr_t ctxt) { }
|
||||
uintptr_t _cgo_wait_runtime_init_done() { }
|
||||
uintptr_t _cgo_wait_runtime_init_done() { return 0; }
|
||||
void _cgo_release_context(uintptr_t ctxt) { }
|
||||
char* _cgo_topofstack(void) { return (char*)0; }
|
||||
void _cgo_allocate(void *a, int c) { }
|
||||
void _cgo_panic(void *a, int c) { }
|
||||
void _cgo_reginit(void) { }
|
||||
|
||||
The extra functions here are stubs to satisfy the references in the C
|
||||
code generated for gcc. The build process links this stub, along with
|
||||
|
|
|
@ -188,21 +188,8 @@ func (p *Package) Translate(f *File) {
|
|||
p.loadDWARF(f, needType)
|
||||
}
|
||||
if p.rewriteCalls(f) {
|
||||
// Add `import _cgo_unsafe "unsafe"` as the first decl
|
||||
// after the package statement.
|
||||
imp := &ast.GenDecl{
|
||||
Tok: token.IMPORT,
|
||||
Specs: []ast.Spec{
|
||||
&ast.ImportSpec{
|
||||
Name: ast.NewIdent("_cgo_unsafe"),
|
||||
Path: &ast.BasicLit{
|
||||
Kind: token.STRING,
|
||||
Value: `"unsafe"`,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
f.AST.Decls = append([]ast.Decl{imp}, f.AST.Decls...)
|
||||
// Add `import _cgo_unsafe "unsafe"` after the package statement.
|
||||
f.Edit.Insert(f.offset(f.AST.Name.End()), "; import _cgo_unsafe \"unsafe\"")
|
||||
}
|
||||
p.rewriteRef(f)
|
||||
}
|
||||
|
@ -211,8 +198,8 @@ func (p *Package) Translate(f *File) {
|
|||
// in the file f and saves relevant renamings in f.Name[name].Define.
|
||||
func (p *Package) loadDefines(f *File) {
|
||||
var b bytes.Buffer
|
||||
b.WriteString(f.Preamble)
|
||||
b.WriteString(builtinProlog)
|
||||
b.WriteString(f.Preamble)
|
||||
stdout := p.gccDefines(b.Bytes())
|
||||
|
||||
for _, line := range strings.Split(stdout, "\n") {
|
||||
|
@ -283,10 +270,6 @@ func (p *Package) guessKinds(f *File) []*Name {
|
|||
if n.IsConst() {
|
||||
continue
|
||||
}
|
||||
|
||||
if isName(n.Define) {
|
||||
n.C = n.Define
|
||||
}
|
||||
}
|
||||
|
||||
// If this is a struct, union, or enum type name, no need to guess the kind.
|
||||
|
@ -315,57 +298,45 @@ func (p *Package) guessKinds(f *File) []*Name {
|
|||
// For each name, we generate these lines, where xxx is the index in toSniff plus one.
|
||||
//
|
||||
// #line xxx "not-declared"
|
||||
// void __cgo_f_xxx_1(void) { __typeof__(name) *__cgo_undefined__; }
|
||||
// void __cgo_f_xxx_1(void) { __typeof__(name) *__cgo_undefined__1; }
|
||||
// #line xxx "not-type"
|
||||
// void __cgo_f_xxx_2(void) { name *__cgo_undefined__; }
|
||||
// void __cgo_f_xxx_2(void) { name *__cgo_undefined__2; }
|
||||
// #line xxx "not-int-const"
|
||||
// void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (name)*1 }; }
|
||||
// void __cgo_f_xxx_3(void) { enum { __cgo_undefined__3 = (name)*1 }; }
|
||||
// #line xxx "not-num-const"
|
||||
// void __cgo_f_xxx_4(void) { static const double x = (name); }
|
||||
// void __cgo_f_xxx_4(void) { static const double __cgo_undefined__4 = (name); }
|
||||
// #line xxx "not-str-lit"
|
||||
// void __cgo_f_xxx_5(void) { static const char x[] = (name); }
|
||||
// #line xxx "not-signed-int-const"
|
||||
// #if 0 < -(name)
|
||||
// #line xxx "not-signed-int-const"
|
||||
// #error found unsigned int
|
||||
// #endif
|
||||
// void __cgo_f_xxx_5(void) { static const char __cgo_undefined__5[] = (name); }
|
||||
//
|
||||
// If we see an error at not-declared:xxx, the corresponding name is not declared.
|
||||
// If we see an error at not-type:xxx, the corresponding name is a type.
|
||||
// If we see an error at not-int-const:xxx, the corresponding name is not an integer constant.
|
||||
// If we see an error at not-num-const:xxx, the corresponding name is not a number constant.
|
||||
// If we see an error at not-str-lit:xxx, the corresponding name is not a string literal.
|
||||
// If we see an error at not-signed-int-const:xxx, the corresponding name is not a signed integer literal.
|
||||
//
|
||||
// The specific input forms are chosen so that they are valid C syntax regardless of
|
||||
// whether name denotes a type or an expression.
|
||||
|
||||
var b bytes.Buffer
|
||||
b.WriteString(f.Preamble)
|
||||
b.WriteString(builtinProlog)
|
||||
b.WriteString(f.Preamble)
|
||||
|
||||
for i, n := range names {
|
||||
fmt.Fprintf(&b, "#line %d \"not-declared\"\n"+
|
||||
"void __cgo_f_%d_1(void) { __typeof__(%s) *__cgo_undefined__; }\n"+
|
||||
"void __cgo_f_%d_1(void) { __typeof__(%s) *__cgo_undefined__1; }\n"+
|
||||
"#line %d \"not-type\"\n"+
|
||||
"void __cgo_f_%d_2(void) { %s *__cgo_undefined__; }\n"+
|
||||
"void __cgo_f_%d_2(void) { %s *__cgo_undefined__2; }\n"+
|
||||
"#line %d \"not-int-const\"\n"+
|
||||
"void __cgo_f_%d_3(void) { enum { __cgo_undefined__ = (%s)*1 }; }\n"+
|
||||
"void __cgo_f_%d_3(void) { enum { __cgo_undefined__3 = (%s)*1 }; }\n"+
|
||||
"#line %d \"not-num-const\"\n"+
|
||||
"void __cgo_f_%d_4(void) { static const double x = (%s); }\n"+
|
||||
"void __cgo_f_%d_4(void) { static const double __cgo_undefined__4 = (%s); }\n"+
|
||||
"#line %d \"not-str-lit\"\n"+
|
||||
"void __cgo_f_%d_5(void) { static const char s[] = (%s); }\n"+
|
||||
"#line %d \"not-signed-int-const\"\n"+
|
||||
"#if 0 < (%s)\n"+
|
||||
"#line %d \"not-signed-int-const\"\n"+
|
||||
"#error found unsigned int\n"+
|
||||
"#endif\n",
|
||||
"void __cgo_f_%d_5(void) { static const char __cgo_undefined__5[] = (%s); }\n",
|
||||
i+1, i+1, n.C,
|
||||
i+1, i+1, n.C,
|
||||
i+1, i+1, n.C,
|
||||
i+1, i+1, n.C,
|
||||
i+1, i+1, n.C,
|
||||
i+1, n.C, i+1,
|
||||
)
|
||||
}
|
||||
fmt.Fprintf(&b, "#line 1 \"completed\"\n"+
|
||||
|
@ -384,7 +355,6 @@ func (p *Package) guessKinds(f *File) []*Name {
|
|||
notNumConst
|
||||
notStrLiteral
|
||||
notDeclared
|
||||
notSignedIntConst
|
||||
)
|
||||
sawUnmatchedErrors := false
|
||||
for _, line := range strings.Split(stderr, "\n") {
|
||||
|
@ -438,8 +408,6 @@ func (p *Package) guessKinds(f *File) []*Name {
|
|||
sniff[i] |= notNumConst
|
||||
case "not-str-lit":
|
||||
sniff[i] |= notStrLiteral
|
||||
case "not-signed-int-const":
|
||||
sniff[i] |= notSignedIntConst
|
||||
default:
|
||||
if isError {
|
||||
sawUnmatchedErrors = true
|
||||
|
@ -455,22 +423,11 @@ func (p *Package) guessKinds(f *File) []*Name {
|
|||
}
|
||||
|
||||
for i, n := range names {
|
||||
switch sniff[i] &^ notSignedIntConst {
|
||||
switch sniff[i] {
|
||||
default:
|
||||
var tpos token.Pos
|
||||
for _, ref := range f.Ref {
|
||||
if ref.Name == n {
|
||||
tpos = ref.Pos()
|
||||
break
|
||||
}
|
||||
}
|
||||
error_(tpos, "could not determine kind of name for C.%s", fixGo(n.Go))
|
||||
error_(f.NamePos[n], "could not determine kind of name for C.%s", fixGo(n.Go))
|
||||
case notStrLiteral | notType:
|
||||
if sniff[i]¬SignedIntConst != 0 {
|
||||
n.Kind = "uconst"
|
||||
} else {
|
||||
n.Kind = "iconst"
|
||||
}
|
||||
n.Kind = "iconst"
|
||||
case notIntConst | notStrLiteral | notType:
|
||||
n.Kind = "fconst"
|
||||
case notIntConst | notNumConst | notType:
|
||||
|
@ -510,12 +467,12 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
|
|||
// for each entry in names and then dereference the type we
|
||||
// learn for __cgo__i.
|
||||
var b bytes.Buffer
|
||||
b.WriteString(f.Preamble)
|
||||
b.WriteString(builtinProlog)
|
||||
b.WriteString(f.Preamble)
|
||||
b.WriteString("#line 1 \"cgo-dwarf-inference\"\n")
|
||||
for i, n := range names {
|
||||
fmt.Fprintf(&b, "__typeof__(%s) *__cgo__%d;\n", n.C, i)
|
||||
if n.Kind == "iconst" || n.Kind == "uconst" {
|
||||
if n.Kind == "iconst" {
|
||||
fmt.Fprintf(&b, "enum { __cgo_enum__%d = %s };\n", i, n.C)
|
||||
}
|
||||
}
|
||||
|
@ -524,7 +481,7 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
|
|||
// so we can read them out of the object file.
|
||||
fmt.Fprintf(&b, "long long __cgodebug_ints[] = {\n")
|
||||
for _, n := range names {
|
||||
if n.Kind == "iconst" || n.Kind == "uconst" {
|
||||
if n.Kind == "iconst" {
|
||||
fmt.Fprintf(&b, "\t%s,\n", n.C)
|
||||
} else {
|
||||
fmt.Fprintf(&b, "\t0,\n")
|
||||
|
@ -562,14 +519,6 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
|
|||
|
||||
// Scan DWARF info for top-level TagVariable entries with AttrName __cgo__i.
|
||||
types := make([]dwarf.Type, len(names))
|
||||
nameToIndex := make(map[*Name]int)
|
||||
for i, n := range names {
|
||||
nameToIndex[n] = i
|
||||
}
|
||||
nameToRef := make(map[*Name]*Ref)
|
||||
for _, ref := range f.Ref {
|
||||
nameToRef[ref.Name] = ref
|
||||
}
|
||||
r := d.Reader()
|
||||
for {
|
||||
e, err := r.Next()
|
||||
|
@ -620,10 +569,7 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
|
|||
if types[i] == nil {
|
||||
continue
|
||||
}
|
||||
pos := token.NoPos
|
||||
if ref, ok := nameToRef[n]; ok {
|
||||
pos = ref.Pos()
|
||||
}
|
||||
pos := f.NamePos[n]
|
||||
f, fok := types[i].(*dwarf.FuncType)
|
||||
if n.Kind != "type" && fok {
|
||||
n.Kind = "func"
|
||||
|
@ -633,11 +579,11 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
|
|||
switch n.Kind {
|
||||
case "iconst":
|
||||
if i < len(ints) {
|
||||
n.Const = fmt.Sprintf("%#x", ints[i])
|
||||
}
|
||||
case "uconst":
|
||||
if i < len(ints) {
|
||||
n.Const = fmt.Sprintf("%#x", uint64(ints[i]))
|
||||
if _, ok := types[i].(*dwarf.UintType); ok {
|
||||
n.Const = fmt.Sprintf("%#x", uint64(ints[i]))
|
||||
} else {
|
||||
n.Const = fmt.Sprintf("%#x", ints[i])
|
||||
}
|
||||
}
|
||||
case "fconst":
|
||||
if i < len(floats) {
|
||||
|
@ -778,8 +724,9 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
|
|||
stmts = append(stmts, stmt)
|
||||
}
|
||||
|
||||
const cgoMarker = "__cgo__###__marker__"
|
||||
fcall := &ast.CallExpr{
|
||||
Fun: call.Call.Fun,
|
||||
Fun: ast.NewIdent(cgoMarker),
|
||||
Args: nargs,
|
||||
}
|
||||
ftype := &ast.FuncType{
|
||||
|
@ -801,31 +748,26 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
|
|||
}
|
||||
}
|
||||
|
||||
// There is a Ref pointing to the old call.Call.Fun.
|
||||
// If this call expects two results, we have to
|
||||
// adjust the results of the function we generated.
|
||||
for _, ref := range f.Ref {
|
||||
if ref.Expr == &call.Call.Fun {
|
||||
ref.Expr = &fcall.Fun
|
||||
|
||||
// If this call expects two results, we have to
|
||||
// adjust the results of the function we generated.
|
||||
if ref.Context == "call2" {
|
||||
if ftype.Results == nil {
|
||||
// An explicit void argument
|
||||
// looks odd but it seems to
|
||||
// be how cgo has worked historically.
|
||||
ftype.Results = &ast.FieldList{
|
||||
List: []*ast.Field{
|
||||
&ast.Field{
|
||||
Type: ast.NewIdent("_Ctype_void"),
|
||||
},
|
||||
if ref.Expr == &call.Call.Fun && ref.Context == ctxCall2 {
|
||||
if ftype.Results == nil {
|
||||
// An explicit void argument
|
||||
// looks odd but it seems to
|
||||
// be how cgo has worked historically.
|
||||
ftype.Results = &ast.FieldList{
|
||||
List: []*ast.Field{
|
||||
&ast.Field{
|
||||
Type: ast.NewIdent("_Ctype_void"),
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
ftype.Results.List = append(ftype.Results.List,
|
||||
&ast.Field{
|
||||
Type: ast.NewIdent("error"),
|
||||
})
|
||||
}
|
||||
ftype.Results.List = append(ftype.Results.List,
|
||||
&ast.Field{
|
||||
Type: ast.NewIdent("error"),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -839,14 +781,16 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
|
|||
Results: []ast.Expr{fcall},
|
||||
}
|
||||
}
|
||||
call.Call.Fun = &ast.FuncLit{
|
||||
lit := &ast.FuncLit{
|
||||
Type: ftype,
|
||||
Body: &ast.BlockStmt{
|
||||
List: append(stmts, fbody),
|
||||
},
|
||||
}
|
||||
call.Call.Lparen = token.NoPos
|
||||
call.Call.Rparen = token.NoPos
|
||||
text := strings.Replace(gofmt(lit), "\n", ";", -1)
|
||||
repl := strings.Split(text, cgoMarker)
|
||||
f.Edit.Insert(f.offset(call.Call.Fun.Pos()), repl[0])
|
||||
f.Edit.Insert(f.offset(call.Call.Fun.End()), repl[1])
|
||||
|
||||
return needsUnsafe
|
||||
}
|
||||
|
@ -1000,8 +944,8 @@ func (p *Package) checkAddrArgs(f *File, args []ast.Expr, x ast.Expr) []ast.Expr
|
|||
// effect is a function call.
|
||||
func (p *Package) hasSideEffects(f *File, x ast.Expr) bool {
|
||||
found := false
|
||||
f.walk(x, "expr",
|
||||
func(f *File, x interface{}, context string) {
|
||||
f.walk(x, ctxExpr,
|
||||
func(f *File, x interface{}, context astContext) {
|
||||
switch x.(type) {
|
||||
case *ast.CallExpr:
|
||||
found = true
|
||||
|
@ -1110,7 +1054,17 @@ func (p *Package) rewriteRef(f *File) {
|
|||
// Assign mangled names.
|
||||
for _, n := range f.Name {
|
||||
if n.Kind == "not-type" {
|
||||
n.Kind = "var"
|
||||
if n.Define == "" {
|
||||
n.Kind = "var"
|
||||
} else {
|
||||
n.Kind = "macro"
|
||||
n.FuncType = &FuncType{
|
||||
Result: n.Type,
|
||||
Go: &ast.FuncType{
|
||||
Results: &ast.FieldList{List: []*ast.Field{{Type: n.Type.Go}}},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
if n.Mangle == "" {
|
||||
p.mangleName(n)
|
||||
|
@ -1130,10 +1084,10 @@ func (p *Package) rewriteRef(f *File) {
|
|||
}
|
||||
var expr ast.Expr = ast.NewIdent(r.Name.Mangle) // default
|
||||
switch r.Context {
|
||||
case "call", "call2":
|
||||
case ctxCall, ctxCall2:
|
||||
if r.Name.Kind != "func" {
|
||||
if r.Name.Kind == "type" {
|
||||
r.Context = "type"
|
||||
r.Context = ctxType
|
||||
if r.Name.Type == nil {
|
||||
error_(r.Pos(), "invalid conversion to C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C)
|
||||
break
|
||||
|
@ -1145,7 +1099,7 @@ func (p *Package) rewriteRef(f *File) {
|
|||
break
|
||||
}
|
||||
functions[r.Name.Go] = true
|
||||
if r.Context == "call2" {
|
||||
if r.Context == ctxCall2 {
|
||||
if r.Name.Go == "_CMalloc" {
|
||||
error_(r.Pos(), "no two-result form for C.malloc")
|
||||
break
|
||||
|
@ -1163,8 +1117,9 @@ func (p *Package) rewriteRef(f *File) {
|
|||
r.Name = n
|
||||
break
|
||||
}
|
||||
case "expr":
|
||||
if r.Name.Kind == "func" {
|
||||
case ctxExpr:
|
||||
switch r.Name.Kind {
|
||||
case "func":
|
||||
if builtinDefs[r.Name.C] != "" {
|
||||
error_(r.Pos(), "use of builtin '%s' not in function call", fixGo(r.Name.C))
|
||||
}
|
||||
|
@ -1191,25 +1146,25 @@ func (p *Package) rewriteRef(f *File) {
|
|||
Fun: &ast.Ident{NamePos: (*r.Expr).Pos(), Name: "_Cgo_ptr"},
|
||||
Args: []ast.Expr{ast.NewIdent(name.Mangle)},
|
||||
}
|
||||
} else if r.Name.Kind == "type" {
|
||||
case "type":
|
||||
// Okay - might be new(T)
|
||||
if r.Name.Type == nil {
|
||||
error_(r.Pos(), "expression C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C)
|
||||
break
|
||||
}
|
||||
expr = r.Name.Type.Go
|
||||
} else if r.Name.Kind == "var" {
|
||||
case "var":
|
||||
expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr}
|
||||
case "macro":
|
||||
expr = &ast.CallExpr{Fun: expr}
|
||||
}
|
||||
|
||||
case "selector":
|
||||
case ctxSelector:
|
||||
if r.Name.Kind == "var" {
|
||||
expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr}
|
||||
} else {
|
||||
error_(r.Pos(), "only C variables allowed in selector expression %s", fixGo(r.Name.Go))
|
||||
}
|
||||
|
||||
case "type":
|
||||
case ctxType:
|
||||
if r.Name.Kind != "type" {
|
||||
error_(r.Pos(), "expression C.%s used as type", fixGo(r.Name.Go))
|
||||
} else if r.Name.Type == nil {
|
||||
|
@ -1224,6 +1179,7 @@ func (p *Package) rewriteRef(f *File) {
|
|||
error_(r.Pos(), "must call C.%s", fixGo(r.Name.Go))
|
||||
}
|
||||
}
|
||||
|
||||
if *godefs {
|
||||
// Substitute definition for mangled type name.
|
||||
if id, ok := expr.(*ast.Ident); ok {
|
||||
|
@ -1245,7 +1201,17 @@ func (p *Package) rewriteRef(f *File) {
|
|||
expr = &ast.Ident{NamePos: pos, Name: x.Name}
|
||||
}
|
||||
|
||||
// Change AST, because some later processing depends on it,
|
||||
// and also because -godefs mode still prints the AST.
|
||||
old := *r.Expr
|
||||
*r.Expr = expr
|
||||
|
||||
// Record source-level edit for cgo output.
|
||||
repl := gofmt(expr)
|
||||
if r.Name.Kind != "type" {
|
||||
repl = "(" + repl + ")"
|
||||
}
|
||||
f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), repl)
|
||||
}
|
||||
|
||||
// Remove functions only used as expressions, so their respective
|
||||
|
@ -1270,7 +1236,7 @@ func (p *Package) gccBaseCmd() []string {
|
|||
if ret := strings.Fields(os.Getenv("GCC")); len(ret) > 0 {
|
||||
return ret
|
||||
}
|
||||
return strings.Fields(defaultCC)
|
||||
return strings.Fields(defaultCC(goos, goarch))
|
||||
}
|
||||
|
||||
// gccMachine returns the gcc -m flag to use, either "-m32", "-m64" or "-marm".
|
||||
|
@ -2186,6 +2152,12 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
|
|||
name := c.Ident("_Ctype_" + dt.Name)
|
||||
goIdent[name.Name] = name
|
||||
sub := c.Type(dt.Type, pos)
|
||||
if badPointerTypedef(dt.Name) {
|
||||
// Treat this typedef as a uintptr.
|
||||
s := *sub
|
||||
s.Go = c.uintptr
|
||||
sub = &s
|
||||
}
|
||||
t.Go = name
|
||||
if unionWithPointer[sub.Go] {
|
||||
unionWithPointer[t.Go] = true
|
||||
|
@ -2266,7 +2238,7 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
|
|||
if ss, ok := dwarfToName[s]; ok {
|
||||
s = ss
|
||||
}
|
||||
s = strings.Join(strings.Split(s, " "), "") // strip spaces
|
||||
s = strings.Replace(s, " ", "", -1)
|
||||
name := c.Ident("_Ctype_" + s)
|
||||
tt := *t
|
||||
typedef[name.Name] = &tt
|
||||
|
@ -2344,6 +2316,17 @@ func (c *typeConv) FuncArg(dtype dwarf.Type, pos token.Pos) *Type {
|
|||
if _, void := base(ptr.Type).(*dwarf.VoidType); void {
|
||||
break
|
||||
}
|
||||
// ...or the typedef is one in which we expect bad pointers.
|
||||
// It will be a uintptr instead of *X.
|
||||
if badPointerTypedef(dt.Name) {
|
||||
break
|
||||
}
|
||||
|
||||
// If we already know the typedef for t just use that.
|
||||
// See issue 19832.
|
||||
if def := typedef[t.Go.(*ast.Ident).Name]; def != nil {
|
||||
break
|
||||
}
|
||||
|
||||
t = c.Type(ptr, pos)
|
||||
if t == nil {
|
||||
|
@ -2500,7 +2483,9 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct
|
|||
size := t.Size
|
||||
talign := t.Align
|
||||
if f.BitSize > 0 {
|
||||
if f.BitSize%8 != 0 {
|
||||
switch f.BitSize {
|
||||
case 8, 16, 32, 64:
|
||||
default:
|
||||
continue
|
||||
}
|
||||
size = f.BitSize / 8
|
||||
|
@ -2676,3 +2661,51 @@ func fieldPrefix(fld []*ast.Field) string {
|
|||
}
|
||||
return prefix
|
||||
}
|
||||
|
||||
// badPointerTypedef reports whether t is a C typedef that should not be considered a pointer in Go.
|
||||
// A typedef is bad if C code sometimes stores non-pointers in this type.
|
||||
// TODO: Currently our best solution is to find these manually and list them as
|
||||
// they come up. A better solution is desired.
|
||||
func badPointerTypedef(t string) bool {
|
||||
// The real bad types are CFNumberRef and CFTypeRef.
|
||||
// Sometimes non-pointers are stored in these types.
|
||||
// CFTypeRef is a supertype of those, so it can have bad pointers in it as well.
|
||||
// We return true for the other CF*Ref types just so casting between them is easier.
|
||||
// See comment below for details about the bad pointers.
|
||||
return goos == "darwin" && strings.HasPrefix(t, "CF") && strings.HasSuffix(t, "Ref")
|
||||
}
|
||||
|
||||
// Comment from Darwin's CFInternal.h
|
||||
/*
|
||||
// Tagged pointer support
|
||||
// Low-bit set means tagged object, next 3 bits (currently)
|
||||
// define the tagged object class, next 4 bits are for type
|
||||
// information for the specific tagged object class. Thus,
|
||||
// the low byte is for type info, and the rest of a pointer
|
||||
// (32 or 64-bit) is for payload, whatever the tagged class.
|
||||
//
|
||||
// Note that the specific integers used to identify the
|
||||
// specific tagged classes can and will change from release
|
||||
// to release (that's why this stuff is in CF*Internal*.h),
|
||||
// as can the definition of type info vs payload above.
|
||||
//
|
||||
#if __LP64__
|
||||
#define CF_IS_TAGGED_OBJ(PTR) ((uintptr_t)(PTR) & 0x1)
|
||||
#define CF_TAGGED_OBJ_TYPE(PTR) ((uintptr_t)(PTR) & 0xF)
|
||||
#else
|
||||
#define CF_IS_TAGGED_OBJ(PTR) 0
|
||||
#define CF_TAGGED_OBJ_TYPE(PTR) 0
|
||||
#endif
|
||||
|
||||
enum {
|
||||
kCFTaggedObjectID_Invalid = 0,
|
||||
kCFTaggedObjectID_Atom = (0 << 1) + 1,
|
||||
kCFTaggedObjectID_Undefined3 = (1 << 1) + 1,
|
||||
kCFTaggedObjectID_Undefined2 = (2 << 1) + 1,
|
||||
kCFTaggedObjectID_Integer = (3 << 1) + 1,
|
||||
kCFTaggedObjectID_DateTS = (4 << 1) + 1,
|
||||
kCFTaggedObjectID_ManagedObjectID = (5 << 1) + 1, // Core Data
|
||||
kCFTaggedObjectID_Date = (6 << 1) + 1,
|
||||
kCFTaggedObjectID_Undefined7 = (7 << 1) + 1,
|
||||
};
|
||||
*/
|
||||
|
|
|
@ -24,6 +24,9 @@ import (
|
|||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"cmd/internal/edit"
|
||||
"cmd/internal/objabi"
|
||||
)
|
||||
|
||||
// A Package collects information about the package we're going to write.
|
||||
|
@ -54,6 +57,12 @@ type File struct {
|
|||
Calls []*Call // all calls to C.xxx in AST
|
||||
ExpFunc []*ExpFunc // exported functions for this file
|
||||
Name map[string]*Name // map from Go name to Name
|
||||
NamePos map[*Name]token.Pos // map from Name to position of the first reference
|
||||
Edit *edit.Buffer
|
||||
}
|
||||
|
||||
func (f *File) offset(p token.Pos) int {
|
||||
return fset.Position(p).Offset
|
||||
}
|
||||
|
||||
func nameKeys(m map[string]*Name) []string {
|
||||
|
@ -75,7 +84,7 @@ type Call struct {
|
|||
type Ref struct {
|
||||
Name *Name
|
||||
Expr *ast.Expr
|
||||
Context string // "type", "expr", "call", or "call2"
|
||||
Context astContext
|
||||
}
|
||||
|
||||
func (r *Ref) Pos() token.Pos {
|
||||
|
@ -88,7 +97,7 @@ type Name struct {
|
|||
Mangle string // name used in generated Go
|
||||
C string // name used in C
|
||||
Define string // #define expansion
|
||||
Kind string // "iconst", "uconst", "fconst", "sconst", "type", "var", "fpvar", "func", "not-type"
|
||||
Kind string // "iconst", "fconst", "sconst", "type", "var", "fpvar", "func", "macro", "not-type"
|
||||
Type *Type // the type of xxx
|
||||
FuncType *FuncType
|
||||
AddError bool
|
||||
|
@ -100,12 +109,12 @@ func (n *Name) IsVar() bool {
|
|||
return n.Kind == "var" || n.Kind == "fpvar"
|
||||
}
|
||||
|
||||
// IsConst reports whether Kind is either "iconst", "uconst", "fconst" or "sconst"
|
||||
// IsConst reports whether Kind is either "iconst", "fconst" or "sconst"
|
||||
func (n *Name) IsConst() bool {
|
||||
return strings.HasSuffix(n.Kind, "const")
|
||||
}
|
||||
|
||||
// A ExpFunc is an exported function, callable from C.
|
||||
// An ExpFunc is an exported function, callable from C.
|
||||
// Such functions are identified in the Go input file
|
||||
// by doc comments containing the line //export ExpName
|
||||
type ExpFunc struct {
|
||||
|
@ -214,6 +223,7 @@ var importSyscall = flag.Bool("import_syscall", true, "import syscall in generat
|
|||
var goarch, goos string
|
||||
|
||||
func main() {
|
||||
objabi.AddVersionFlag() // -V
|
||||
flag.Usage = usage
|
||||
flag.Parse()
|
||||
|
||||
|
@ -294,6 +304,7 @@ func main() {
|
|||
}
|
||||
|
||||
f := new(File)
|
||||
f.Edit = edit.NewBuffer(b)
|
||||
f.ParseGo(input, b)
|
||||
f.DiscardCgoDirectives()
|
||||
fs[i] = f
|
||||
|
@ -314,11 +325,13 @@ func main() {
|
|||
p.Translate(f)
|
||||
for _, cref := range f.Ref {
|
||||
switch cref.Context {
|
||||
case "call", "call2":
|
||||
case ctxCall, ctxCall2:
|
||||
if cref.Name.Kind != "type" {
|
||||
break
|
||||
}
|
||||
old := *cref.Expr
|
||||
*cref.Expr = cref.Name.Type.Go
|
||||
f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
|
||||
}
|
||||
}
|
||||
if nerrors > 0 {
|
||||
|
|
|
@ -16,6 +16,7 @@ import (
|
|||
"go/token"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
@ -116,7 +117,13 @@ func (p *Package) writeDefs() {
|
|||
// Which is not useful. Moreover we never override source info,
|
||||
// so subsequent source code uses the same source info.
|
||||
// Moreover, empty file name makes compile emit no source debug info at all.
|
||||
noSourceConf.Fprint(fgo2, fset, def.Go)
|
||||
var buf bytes.Buffer
|
||||
noSourceConf.Fprint(&buf, fset, def.Go)
|
||||
if bytes.HasPrefix(buf.Bytes(), []byte("_Ctype_")) {
|
||||
// This typedef is of the form `typedef a b` and should be an alias.
|
||||
fmt.Fprintf(fgo2, "= ")
|
||||
}
|
||||
fmt.Fprintf(fgo2, "%s", buf.Bytes())
|
||||
fmt.Fprintf(fgo2, "\n\n")
|
||||
}
|
||||
if *gccgo {
|
||||
|
@ -424,10 +431,12 @@ func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name, callsMalloc *bool) {
|
|||
inProlog := builtinDefs[name] != ""
|
||||
cname := fmt.Sprintf("_cgo%s%s", cPrefix, n.Mangle)
|
||||
paramnames := []string(nil)
|
||||
for i, param := range d.Type.Params.List {
|
||||
paramName := fmt.Sprintf("p%d", i)
|
||||
param.Names = []*ast.Ident{ast.NewIdent(paramName)}
|
||||
paramnames = append(paramnames, paramName)
|
||||
if d.Type.Params != nil {
|
||||
for i, param := range d.Type.Params.List {
|
||||
paramName := fmt.Sprintf("p%d", i)
|
||||
param.Names = []*ast.Ident{ast.NewIdent(paramName)}
|
||||
paramnames = append(paramnames, paramName)
|
||||
}
|
||||
}
|
||||
|
||||
if *gccgo {
|
||||
|
@ -526,8 +535,10 @@ func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name, callsMalloc *bool) {
|
|||
fmt.Fprintf(fgo2, "\tif errno != 0 { r2 = syscall.Errno(errno) }\n")
|
||||
}
|
||||
fmt.Fprintf(fgo2, "\tif _Cgo_always_false {\n")
|
||||
for i := range d.Type.Params.List {
|
||||
fmt.Fprintf(fgo2, "\t\t_Cgo_use(p%d)\n", i)
|
||||
if d.Type.Params != nil {
|
||||
for i := range d.Type.Params.List {
|
||||
fmt.Fprintf(fgo2, "\t\t_Cgo_use(p%d)\n", i)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(fgo2, "\t}\n")
|
||||
fmt.Fprintf(fgo2, "\treturn\n")
|
||||
|
@ -540,7 +551,7 @@ func (p *Package) writeOutput(f *File, srcfile string) {
|
|||
if strings.HasSuffix(base, ".go") {
|
||||
base = base[0 : len(base)-3]
|
||||
}
|
||||
base = strings.Map(slashToUnderscore, base)
|
||||
base = filepath.Base(base)
|
||||
fgo1 := creat(*objDir + base + ".cgo1.go")
|
||||
fgcc := creat(*objDir + base + ".cgo2.c")
|
||||
|
||||
|
@ -549,10 +560,12 @@ func (p *Package) writeOutput(f *File, srcfile string) {
|
|||
|
||||
// Write Go output: Go input with rewrites of C.xxx to _C_xxx.
|
||||
fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n\n")
|
||||
conf.Fprint(fgo1, fset, f.AST)
|
||||
fmt.Fprintf(fgo1, "//line %s:1\n", srcfile)
|
||||
fgo1.Write(f.Edit.Bytes())
|
||||
|
||||
// While we process the vars and funcs, also write gcc output.
|
||||
// Gcc output starts with the preamble.
|
||||
fmt.Fprintf(fgcc, "%s\n", builtinProlog)
|
||||
fmt.Fprintf(fgcc, "%s\n", f.Preamble)
|
||||
fmt.Fprintf(fgcc, "%s\n", gccProlog)
|
||||
fmt.Fprintf(fgcc, "%s\n", tsanProlog)
|
||||
|
@ -639,14 +652,18 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
|
|||
fmt.Fprint(fgcc, "(__typeof__(a->r)) ")
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(fgcc, "%s(", n.C)
|
||||
for i := range n.FuncType.Params {
|
||||
if i > 0 {
|
||||
fmt.Fprintf(fgcc, ", ")
|
||||
if n.Kind == "macro" {
|
||||
fmt.Fprintf(fgcc, "%s;\n", n.C)
|
||||
} else {
|
||||
fmt.Fprintf(fgcc, "%s(", n.C)
|
||||
for i := range n.FuncType.Params {
|
||||
if i > 0 {
|
||||
fmt.Fprintf(fgcc, ", ")
|
||||
}
|
||||
fmt.Fprintf(fgcc, "a->p%d", i)
|
||||
}
|
||||
fmt.Fprintf(fgcc, "a->p%d", i)
|
||||
fmt.Fprintf(fgcc, ");\n")
|
||||
}
|
||||
fmt.Fprintf(fgcc, ");\n")
|
||||
if n.AddError {
|
||||
fmt.Fprintf(fgcc, "\t_cgo_errno = errno;\n")
|
||||
}
|
||||
|
@ -702,14 +719,18 @@ func (p *Package) writeGccgoOutputFunc(fgcc *os.File, n *Name) {
|
|||
fmt.Fprintf(fgcc, "(void*)")
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(fgcc, "%s(", n.C)
|
||||
for i := range n.FuncType.Params {
|
||||
if i > 0 {
|
||||
fmt.Fprintf(fgcc, ", ")
|
||||
if n.Kind == "macro" {
|
||||
fmt.Fprintf(fgcc, "%s;\n", n.C)
|
||||
} else {
|
||||
fmt.Fprintf(fgcc, "%s(", n.C)
|
||||
for i := range n.FuncType.Params {
|
||||
if i > 0 {
|
||||
fmt.Fprintf(fgcc, ", ")
|
||||
}
|
||||
fmt.Fprintf(fgcc, "p%d", i)
|
||||
}
|
||||
fmt.Fprintf(fgcc, "p%d", i)
|
||||
fmt.Fprintf(fgcc, ");\n")
|
||||
}
|
||||
fmt.Fprintf(fgcc, ");\n")
|
||||
fmt.Fprintf(fgcc, "\t_cgo_tsan_release();\n")
|
||||
if t := n.FuncType.Result; t != nil {
|
||||
fmt.Fprintf(fgcc, "\treturn ")
|
||||
|
@ -1009,7 +1030,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
|
|||
default:
|
||||
// Declare a result struct.
|
||||
fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName)
|
||||
fmt.Fprintf(fgcch, "struct %s_result {\n", exp.ExpName)
|
||||
fmt.Fprintf(fgcch, "struct %s_return {\n", exp.ExpName)
|
||||
forFieldList(fntype.Results,
|
||||
func(i int, aname string, atype ast.Expr) {
|
||||
t := p.cgoType(atype)
|
||||
|
@ -1020,7 +1041,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
|
|||
fmt.Fprint(fgcch, "\n")
|
||||
})
|
||||
fmt.Fprintf(fgcch, "};\n")
|
||||
fmt.Fprintf(cdeclBuf, "struct %s_result", exp.ExpName)
|
||||
fmt.Fprintf(cdeclBuf, "struct %s_return", exp.ExpName)
|
||||
}
|
||||
|
||||
cRet := cdeclBuf.String()
|
||||
|
@ -1046,7 +1067,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
|
|||
fmt.Fprintf(fgcch, "\n%s", exp.Doc)
|
||||
}
|
||||
|
||||
fmt.Fprintf(fgcch, "extern %s %s %s;\n", cRet, exp.ExpName, cParams)
|
||||
fmt.Fprintf(fgcch, "extern %s %s%s;\n", cRet, exp.ExpName, cParams)
|
||||
|
||||
// We need to use a name that will be exported by the
|
||||
// Go code; otherwise gccgo will make it static and we
|
||||
|
@ -1155,6 +1176,7 @@ func (p *Package) writeExportHeader(fgcch io.Writer) {
|
|||
pkg = p.PackagePath
|
||||
}
|
||||
fmt.Fprintf(fgcch, "/* package %s */\n\n", pkg)
|
||||
fmt.Fprintf(fgcch, "%s\n", builtinExportProlog)
|
||||
|
||||
fmt.Fprintf(fgcch, "/* Start of preamble from import \"C\" comments. */\n\n")
|
||||
fmt.Fprintf(fgcch, "%s\n", p.Preamble)
|
||||
|
@ -1247,8 +1269,9 @@ func (p *Package) cgoType(e ast.Expr) *Type {
|
|||
// Slice: pointer, len, cap.
|
||||
return &Type{Size: p.PtrSize * 3, Align: p.PtrSize, C: c("GoSlice")}
|
||||
}
|
||||
// Non-slice array types are not supported.
|
||||
case *ast.StructType:
|
||||
// TODO
|
||||
// Not supported.
|
||||
case *ast.FuncType:
|
||||
return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("void*")}
|
||||
case *ast.InterfaceType:
|
||||
|
@ -1398,7 +1421,7 @@ const builtinProlog = `
|
|||
/* Define intgo when compiling with GCC. */
|
||||
typedef ptrdiff_t intgo;
|
||||
|
||||
typedef struct { char *p; intgo n; } _GoString_;
|
||||
typedef struct { const char *p; intgo n; } _GoString_;
|
||||
typedef struct { char *p; intgo n; intgo c; } _GoBytes_;
|
||||
_GoString_ GoString(char *p);
|
||||
_GoString_ GoStringN(char *p, int l);
|
||||
|
@ -1406,6 +1429,12 @@ _GoBytes_ GoBytes(void *p, int n);
|
|||
char *CString(_GoString_);
|
||||
void *CBytes(_GoBytes_);
|
||||
void *_CMalloc(size_t);
|
||||
|
||||
__attribute__ ((unused))
|
||||
static size_t _GoStringLen(_GoString_ s) { return s.n; }
|
||||
|
||||
__attribute__ ((unused))
|
||||
static const char *_GoStringPtr(_GoString_ s) { return s.p; }
|
||||
`
|
||||
|
||||
const goProlog = `
|
||||
|
@ -1637,6 +1666,27 @@ void localCgoCheckResult(Eface val) {
|
|||
}
|
||||
`
|
||||
|
||||
// builtinExportProlog is a shorter version of builtinProlog,
|
||||
// to be put into the _cgo_export.h file.
|
||||
// For historical reasons we can't use builtinProlog in _cgo_export.h,
|
||||
// because _cgo_export.h defines GoString as a struct while builtinProlog
|
||||
// defines it as a function. We don't change this to avoid unnecessarily
|
||||
// breaking existing code.
|
||||
const builtinExportProlog = `
|
||||
#line 1 "cgo-builtin-prolog"
|
||||
|
||||
#include <stddef.h> /* for ptrdiff_t below */
|
||||
|
||||
#ifndef GO_CGO_EXPORT_PROLOGUE_H
|
||||
#define GO_CGO_EXPORT_PROLOGUE_H
|
||||
|
||||
typedef ptrdiff_t intgo;
|
||||
|
||||
typedef struct { const char *p; intgo n; } _GoString_;
|
||||
|
||||
#endif
|
||||
`
|
||||
|
||||
func (p *Package) gccExportHeaderProlog() string {
|
||||
return strings.Replace(gccExportHeaderProlog, "GOINTBITS", fmt.Sprint(8*p.IntSize), -1)
|
||||
}
|
||||
|
@ -1670,7 +1720,7 @@ typedef double _Complex GoComplex128;
|
|||
*/
|
||||
typedef char _check_for_GOINTBITS_bit_pointer_matching_GoInt[sizeof(void*)==GOINTBITS/8 ? 1:-1];
|
||||
|
||||
typedef struct { const char *p; GoInt n; } GoString;
|
||||
typedef _GoString_ GoString;
|
||||
typedef void *GoMap;
|
||||
typedef void *GoChan;
|
||||
typedef struct { void *t; void *v; } GoInterface;
|
||||
|
|
|
@ -14,12 +14,12 @@
|
|||
// The commands are:
|
||||
//
|
||||
// build compile packages and dependencies
|
||||
// clean remove object files
|
||||
// clean remove object files and cached files
|
||||
// doc show documentation for package or symbol
|
||||
// env print Go environment information
|
||||
// bug start a bug report
|
||||
// fix run go tool fix on packages
|
||||
// fmt run gofmt on package sources
|
||||
// fix update packages to use new APIs
|
||||
// fmt gofmt (reformat) package sources
|
||||
// generate generate Go files by processing source
|
||||
// get download and install packages and dependencies
|
||||
// install compile and install packages and dependencies
|
||||
|
@ -28,7 +28,7 @@
|
|||
// test test packages
|
||||
// tool run specified go tool
|
||||
// version print Go version
|
||||
// vet run go tool vet on packages
|
||||
// vet report likely mistakes in packages
|
||||
//
|
||||
// Use "go help [command]" for more information about a command.
|
||||
//
|
||||
|
@ -104,15 +104,15 @@
|
|||
// -x
|
||||
// print the commands.
|
||||
//
|
||||
// -asmflags 'flag list'
|
||||
// -asmflags '[pattern=]arg list'
|
||||
// arguments to pass on each go tool asm invocation.
|
||||
// -buildmode mode
|
||||
// build mode to use. See 'go help buildmode' for more.
|
||||
// -compiler name
|
||||
// name of compiler to use, as in runtime.Compiler (gccgo or gc).
|
||||
// -gccgoflags 'arg list'
|
||||
// -gccgoflags '[pattern=]arg list'
|
||||
// arguments to pass on each gccgo compiler/linker invocation.
|
||||
// -gcflags 'arg list'
|
||||
// -gcflags '[pattern=]arg list'
|
||||
// arguments to pass on each go tool compile invocation.
|
||||
// -installsuffix suffix
|
||||
// a suffix to use in the name of the package installation directory,
|
||||
|
@ -121,7 +121,7 @@
|
|||
// or, if set explicitly, has _race appended to it. Likewise for the -msan
|
||||
// flag. Using a -buildmode option that requires non-default compile flags
|
||||
// has a similar effect.
|
||||
// -ldflags 'flag list'
|
||||
// -ldflags '[pattern=]arg list'
|
||||
// arguments to pass on each go tool link invocation.
|
||||
// -linkshared
|
||||
// link against shared libraries previously created with
|
||||
|
@ -139,9 +139,21 @@
|
|||
// For example, instead of running asm, the go command will run
|
||||
// 'cmd args /path/to/asm <arguments for asm>'.
|
||||
//
|
||||
// All the flags that take a list of arguments accept a space-separated
|
||||
// list of strings. To embed spaces in an element in the list, surround
|
||||
// it with either single or double quotes.
|
||||
// The -asmflags, -gccgoflags, -gcflags, and -ldflags flags accept a
|
||||
// space-separated list of arguments to pass to an underlying tool
|
||||
// during the build. To embed spaces in an element in the list, surround
|
||||
// it with either single or double quotes. The argument list may be
|
||||
// preceded by a package pattern and an equal sign, which restricts
|
||||
// the use of that argument list to the building of packages matching
|
||||
// that pattern (see 'go help packages' for a description of package
|
||||
// patterns). Without a pattern, the argument list applies only to the
|
||||
// packages named on the command line. The flags may be repeated
|
||||
// with different patterns in order to specify different arguments for
|
||||
// different sets of packages. If a package matches patterns given in
|
||||
// multiple flags, the latest match on the command line wins.
|
||||
// For example, 'go build -gcflags=-S fmt' prints the disassembly
|
||||
// only for package fmt, while 'go build -gcflags=all=-S fmt'
|
||||
// prints the disassembly for fmt and all its dependencies.
|
||||
//
|
||||
// For more about specifying packages, see 'go help packages'.
|
||||
// For more about where packages and binaries are installed,
|
||||
|
@ -158,11 +170,11 @@
|
|||
// See also: go install, go get, go clean.
|
||||
//
|
||||
//
|
||||
// Remove object files
|
||||
// Remove object files and cached files
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// go clean [-i] [-r] [-n] [-x] [build flags] [packages]
|
||||
// go clean [-i] [-r] [-n] [-x] [-cache] [-testcache] [build flags] [packages]
|
||||
//
|
||||
// Clean removes object files from package source directories.
|
||||
// The go command builds most objects in a temporary directory,
|
||||
|
@ -200,6 +212,11 @@
|
|||
//
|
||||
// The -x flag causes clean to print remove commands as it executes them.
|
||||
//
|
||||
// The -cache flag causes clean to remove the entire go build cache.
|
||||
//
|
||||
// The -testcache flag causes clean to expire all test results in the
|
||||
// go build cache.
|
||||
//
|
||||
// For more about build flags, see 'go help build'.
|
||||
//
|
||||
// For more about specifying packages, see 'go help packages'.
|
||||
|
@ -328,6 +345,8 @@
|
|||
// The -json flag prints the environment in JSON format
|
||||
// instead of as a shell script.
|
||||
//
|
||||
// For more about environment variables, see 'go help environment'.
|
||||
//
|
||||
//
|
||||
// Start a bug report
|
||||
//
|
||||
|
@ -339,7 +358,7 @@
|
|||
// The report includes useful system information.
|
||||
//
|
||||
//
|
||||
// Run go tool fix on packages
|
||||
// Update packages to use new APIs
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
|
@ -355,7 +374,7 @@
|
|||
// See also: go fmt, go vet.
|
||||
//
|
||||
//
|
||||
// Run gofmt on package sources
|
||||
// Gofmt (reformat) package sources
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
|
@ -543,10 +562,11 @@
|
|||
//
|
||||
// Usage:
|
||||
//
|
||||
// go install [build flags] [packages]
|
||||
// go install [-i] [build flags] [packages]
|
||||
//
|
||||
// Install compiles and installs the packages named by the import paths,
|
||||
// along with their dependencies.
|
||||
// Install compiles and installs the packages named by the import paths.
|
||||
//
|
||||
// The -i flag installs the dependencies of the named packages as well.
|
||||
//
|
||||
// For more about the build flags, see 'go help build'.
|
||||
// For more about specifying packages, see 'go help packages'.
|
||||
|
@ -719,10 +739,10 @@
|
|||
//
|
||||
// 'Go test' recompiles each package along with any files with names matching
|
||||
// the file pattern "*_test.go".
|
||||
// Files whose names begin with "_" (including "_test.go") or "." are ignored.
|
||||
// These additional files can contain test functions, benchmark functions, and
|
||||
// example functions. See 'go help testfunc' for more.
|
||||
// Each listed package causes the execution of a separate test binary.
|
||||
// Files whose names begin with "_" (including "_test.go") or "." are ignored.
|
||||
//
|
||||
// Test files that declare a package with the suffix "_test" will be compiled as a
|
||||
// separate package, and then linked and run with the main test binary.
|
||||
|
@ -730,11 +750,46 @@
|
|||
// The go tool will ignore a directory named "testdata", making it available
|
||||
// to hold ancillary data needed by the tests.
|
||||
//
|
||||
// By default, go test needs no arguments. It compiles and tests the package
|
||||
// with source in the current directory, including tests, and runs the tests.
|
||||
// As part of building a test binary, go test runs go vet on the package
|
||||
// and its test source files to identify significant problems. If go vet
|
||||
// finds any problems, go test reports those and does not run the test binary.
|
||||
// Only a high-confidence subset of the default go vet checks are used.
|
||||
// To disable the running of go vet, use the -vet=off flag.
|
||||
//
|
||||
// The package is built in a temporary directory so it does not interfere with the
|
||||
// non-test installation.
|
||||
// Go test runs in two different modes: local directory mode when invoked with
|
||||
// no package arguments (for example, 'go test'), and package list mode when
|
||||
// invoked with package arguments (for example 'go test math', 'go test ./...',
|
||||
// and even 'go test .').
|
||||
//
|
||||
// In local directory mode, go test compiles and tests the package sources
|
||||
// found in the current directory and then runs the resulting test binary.
|
||||
// In this mode, caching (discussed below) is disabled. After the package test
|
||||
// finishes, go test prints a summary line showing the test status ('ok' or 'FAIL'),
|
||||
// package name, and elapsed time.
|
||||
//
|
||||
// In package list mode, go test compiles and tests each of the packages
|
||||
// listed on the command line. If a package test passes, go test prints only
|
||||
// the final 'ok' summary line. If a package test fails, go test prints the
|
||||
// full test output. If invoked with the -bench or -v flag, go test prints
|
||||
// the full output even for passing package tests, in order to display the
|
||||
// requested benchmark results or verbose logging.
|
||||
//
|
||||
// All test output and summary lines are printed to the go command's standard
|
||||
// output, even if the test printed them to its own standard error.
|
||||
// (The go command's standard error is reserved for printing errors building
|
||||
// the tests.)
|
||||
//
|
||||
// In package list mode, go test also caches successful package test results.
|
||||
// If go test has cached a previous test run using the same test binary and
|
||||
// the same command line consisting entirely of cacheable test flags
|
||||
// (defined as -cpu, -list, -parallel, -run, -short, and -v),
|
||||
// go test will redisplay the previous output instead of running the test
|
||||
// binary again. In the summary line, go test prints '(cached)' in place of
|
||||
// the elapsed time. To disable test caching, use any test flag or argument
|
||||
// other than the cacheable flags. The idiomatic way to disable test caching
|
||||
// explicitly is to use -count=1. A cached result is treated as executing in
|
||||
// no time at all, so a successful package test result will be cached and reused
|
||||
// regardless of -timeout setting.
|
||||
//
|
||||
// In addition to the build flags, the flags handled by 'go test' itself are:
|
||||
//
|
||||
|
@ -757,6 +812,10 @@
|
|||
// Install packages that are dependencies of the test.
|
||||
// Do not run the test.
|
||||
//
|
||||
// -json
|
||||
// Convert test output to JSON suitable for automated processing.
|
||||
// See 'go doc test2json' for the encoding details.
|
||||
//
|
||||
// -o file
|
||||
// Compile the test binary to the named file.
|
||||
// The test still runs (unless -c or -i is specified).
|
||||
|
@ -782,7 +841,7 @@
|
|||
// The -n flag causes tool to print the command that would be
|
||||
// executed but not execute it.
|
||||
//
|
||||
// For more about each tool command, see 'go tool command -h'.
|
||||
// For more about each tool command, see 'go doc cmd/<command>'.
|
||||
//
|
||||
//
|
||||
// Print Go version
|
||||
|
@ -794,7 +853,7 @@
|
|||
// Version prints the Go version, as reported by runtime.Version.
|
||||
//
|
||||
//
|
||||
// Run go tool vet on packages
|
||||
// Report likely mistakes in packages
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
|
@ -808,7 +867,9 @@
|
|||
// The -n flag prints commands that would be executed.
|
||||
// The -x flag prints commands as they are executed.
|
||||
//
|
||||
// For more about build flags, see 'go help build'.
|
||||
// The build flags supported by go vet are those that control package resolution
|
||||
// and execution, such as -n, -x, -v, -tags, and -toolexec.
|
||||
// For more about these flags, see 'go help build'.
|
||||
//
|
||||
// See also: go fmt, go fix.
|
||||
//
|
||||
|
@ -917,8 +978,10 @@
|
|||
// comment, indicating that the package sources are included
|
||||
// for documentation only and must not be used to build the
|
||||
// package binary. This enables distribution of Go packages in
|
||||
// their compiled form alone. See the go/build package documentation
|
||||
// for more details.
|
||||
// their compiled form alone. Even binary-only packages require
|
||||
// accurate import blocks listing required dependencies, so that
|
||||
// those dependencies can be supplied when linking the resulting
|
||||
// command.
|
||||
//
|
||||
//
|
||||
// GOPATH environment variable
|
||||
|
@ -1096,6 +1159,12 @@
|
|||
// See https://golang.org/doc/articles/race_detector.html.
|
||||
// GOROOT
|
||||
// The root of the go tree.
|
||||
// GOTMPDIR
|
||||
// The directory where the go command will write
|
||||
// temporary source files, packages, and binaries.
|
||||
// GOCACHE
|
||||
// The directory where the go command will store
|
||||
// cached information for reuse in future builds.
|
||||
//
|
||||
// Environment variables for use with cgo:
|
||||
//
|
||||
|
@ -1130,6 +1199,9 @@
|
|||
// GO386
|
||||
// For GOARCH=386, the floating point instruction set.
|
||||
// Valid values are 387, sse2.
|
||||
// GOMIPS
|
||||
// For GOARCH=mips{,le}, whether to use floating point instructions.
|
||||
// Valid values are hardfloat (default), softfloat.
|
||||
//
|
||||
// Special-purpose environment variables:
|
||||
//
|
||||
|
@ -1460,10 +1532,10 @@
|
|||
// significantly more expensive.
|
||||
// Sets -cover.
|
||||
//
|
||||
// -coverpkg pkg1,pkg2,pkg3
|
||||
// Apply coverage analysis in each test to the given list of packages.
|
||||
// -coverpkg pattern1,pattern2,pattern3
|
||||
// Apply coverage analysis in each test to packages matching the patterns.
|
||||
// The default is for each test to analyze only the package being tested.
|
||||
// Packages are specified as import paths.
|
||||
// See 'go help packages' for a description of package patterns.
|
||||
// Sets -cover.
|
||||
//
|
||||
// -cpu 1,2,4
|
||||
|
@ -1471,6 +1543,9 @@
|
|||
// benchmarks should be executed. The default is the current value
|
||||
// of GOMAXPROCS.
|
||||
//
|
||||
// -failfast
|
||||
// Do not start new tests after the first test failure.
|
||||
//
|
||||
// -list regexp
|
||||
// List tests, benchmarks, or examples matching the regular expression.
|
||||
// No tests, benchmarks or examples will be run. This will only
|
||||
|
@ -1503,12 +1578,20 @@
|
|||
//
|
||||
// -timeout d
|
||||
// If a test binary runs longer than duration d, panic.
|
||||
// If d is 0, the timeout is disabled.
|
||||
// The default is 10 minutes (10m).
|
||||
//
|
||||
// -v
|
||||
// Verbose output: log all tests as they are run. Also print all
|
||||
// text from Log and Logf calls even if the test succeeds.
|
||||
//
|
||||
// -vet list
|
||||
// Configure the invocation of "go vet" during "go test"
|
||||
// to use the comma-separated list of vet checks.
|
||||
// If list is empty, "go test" runs "go vet" with a curated list of
|
||||
// checks believed to be always worth addressing.
|
||||
// If list is "off", "go test" does not run "go vet" at all.
|
||||
//
|
||||
// The following flags are also recognized by 'go test' and can be used to
|
||||
// profile the tests during execution:
|
||||
//
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -5,12 +5,14 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"internal/testenv"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
@ -54,3 +56,82 @@ func TestAbsolutePath(t *testing.T) {
|
|||
t.Fatalf("wrong output found: %v %v", err, string(output))
|
||||
}
|
||||
}
|
||||
|
||||
func isWindowsXP(t *testing.T) bool {
|
||||
v, err := syscall.GetVersion()
|
||||
if err != nil {
|
||||
t.Fatalf("GetVersion failed: %v", err)
|
||||
}
|
||||
major := byte(v)
|
||||
return major < 6
|
||||
}
|
||||
|
||||
func runIcacls(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("icacls", args...).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("icacls failed: %v\n%v", err, string(out))
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func runGetACL(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
cmd := fmt.Sprintf(`Get-Acl "%s" | Select -expand AccessToString`, path)
|
||||
out, err := exec.Command("powershell", "-Command", cmd).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("Get-Acl failed: %v\n%v", err, string(out))
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// For issue 22343: verify that executable file created by "go build" command
|
||||
// has discretionary access control list (DACL) set as if the file
|
||||
// was created in the destination directory.
|
||||
func TestACL(t *testing.T) {
|
||||
if isWindowsXP(t) {
|
||||
t.Skip("Windows XP does not have powershell command")
|
||||
}
|
||||
|
||||
tmpdir, err := ioutil.TempDir("", "TestACL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
|
||||
newtmpdir := filepath.Join(tmpdir, "tmp")
|
||||
err = os.Mkdir(newtmpdir, 0777)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// When TestACL/tmp directory is created, it will have
|
||||
// the same security attributes as TestACL.
|
||||
// Add Guest account full access to TestACL/tmp - this
|
||||
// will make all files created in TestACL/tmp have different
|
||||
// security attributes to the files created in TestACL.
|
||||
runIcacls(t, newtmpdir,
|
||||
"/grant", "guest:(oi)(ci)f", // add Guest user to have full access
|
||||
)
|
||||
|
||||
src := filepath.Join(tmpdir, "main.go")
|
||||
err = ioutil.WriteFile(src, []byte("package main; func main() { }\n"), 0644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exe := filepath.Join(tmpdir, "main.exe")
|
||||
cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", exe, src)
|
||||
cmd.Env = append(os.Environ(),
|
||||
"TMP="+newtmpdir,
|
||||
"TEMP="+newtmpdir,
|
||||
)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("go command failed: %v\n%v", err, string(out))
|
||||
}
|
||||
|
||||
// exe file is expected to have the same security attributes as the src.
|
||||
if got, expected := runGetACL(t, exe), runGetACL(t, src); got != expected {
|
||||
t.Fatalf("expected Get-Acl output of \n%v\n, got \n%v\n", expected, got)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -62,8 +62,8 @@ func (c *Command) Name() string {
|
|||
}
|
||||
|
||||
func (c *Command) Usage() {
|
||||
fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.UsageLine)
|
||||
fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(c.Long))
|
||||
fmt.Fprintf(os.Stderr, "usage: %s\n", c.UsageLine)
|
||||
fmt.Fprintf(os.Stderr, "Run 'go help %s' for details.\n", c.Name())
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
|
|
|
@ -44,28 +44,6 @@ func RelPaths(paths []string) []string {
|
|||
return out
|
||||
}
|
||||
|
||||
// FilterDotUnderscoreFiles returns a slice containing all elements
|
||||
// of path whose base name doesn't begin with "." or "_".
|
||||
func FilterDotUnderscoreFiles(path []string) []string {
|
||||
var out []string // lazily initialized
|
||||
for i, p := range path {
|
||||
base := filepath.Base(p)
|
||||
if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") {
|
||||
if out == nil {
|
||||
out = append(make([]string, 0, len(path)), path[:i]...)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if out != nil {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
if out == nil {
|
||||
return path
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// IsTestFile reports whether the source file is a set of tests and should therefore
|
||||
// be excluded from coverage analysis.
|
||||
func IsTestFile(file string) bool {
|
||||
|
|
|
@ -36,18 +36,9 @@ func Tool(toolName string) string {
|
|||
}
|
||||
// Give a nice message if there is no tool with that name.
|
||||
if _, err := os.Stat(toolPath); err != nil {
|
||||
if isInGoToolsRepo(toolName) {
|
||||
fmt.Fprintf(os.Stderr, "go tool: no such tool %q; to install:\n\tgo get golang.org/x/tools/cmd/%s\n", toolName, toolName)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName)
|
||||
SetExitStatus(2)
|
||||
Exit()
|
||||
}
|
||||
return toolPath
|
||||
}
|
||||
|
||||
// TODO: Delete.
|
||||
func isInGoToolsRepo(toolName string) bool {
|
||||
return false
|
||||
}
|
||||
|
|
|
@ -0,0 +1,453 @@
|
|||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package cache implements a build artifact cache.
|
||||
package cache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// An ActionID is a cache action key, the hash of a complete description of a
|
||||
// repeatable computation (command line, environment variables,
|
||||
// input file contents, executable contents).
|
||||
type ActionID [HashSize]byte
|
||||
|
||||
// An OutputID is a cache output key, the hash of an output of a computation.
|
||||
type OutputID [HashSize]byte
|
||||
|
||||
// A Cache is a package cache, backed by a file system directory tree.
|
||||
type Cache struct {
|
||||
dir string
|
||||
log *os.File
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// Open opens and returns the cache in the given directory.
|
||||
//
|
||||
// It is safe for multiple processes on a single machine to use the
|
||||
// same cache directory in a local file system simultaneously.
|
||||
// They will coordinate using operating system file locks and may
|
||||
// duplicate effort but will not corrupt the cache.
|
||||
//
|
||||
// However, it is NOT safe for multiple processes on different machines
|
||||
// to share a cache directory (for example, if the directory were stored
|
||||
// in a network file system). File locking is notoriously unreliable in
|
||||
// network file systems and may not suffice to protect the cache.
|
||||
//
|
||||
func Open(dir string) (*Cache, error) {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, &os.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")}
|
||||
}
|
||||
for i := 0; i < 256; i++ {
|
||||
name := filepath.Join(dir, fmt.Sprintf("%02x", i))
|
||||
if err := os.MkdirAll(name, 0777); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
f, err := os.OpenFile(filepath.Join(dir, "log.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &Cache{
|
||||
dir: dir,
|
||||
log: f,
|
||||
now: time.Now,
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// fileName returns the name of the file corresponding to the given id.
|
||||
func (c *Cache) fileName(id [HashSize]byte, key string) string {
|
||||
return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key)
|
||||
}
|
||||
|
||||
var errMissing = errors.New("cache entry not found")
|
||||
|
||||
const (
|
||||
// action entry file is "v1 <hex id> <hex out> <decimal size space-padded to 20 bytes> <unixnano space-padded to 20 bytes>\n"
|
||||
hexSize = HashSize * 2
|
||||
entrySize = 2 + 1 + hexSize + 1 + hexSize + 1 + 20 + 1 + 20 + 1
|
||||
)
|
||||
|
||||
// verify controls whether to run the cache in verify mode.
|
||||
// In verify mode, the cache always returns errMissing from Get
|
||||
// but then double-checks in Put that the data being written
|
||||
// exactly matches any existing entry. This provides an easy
|
||||
// way to detect program behavior that would have been different
|
||||
// had the cache entry been returned from Get.
|
||||
//
|
||||
// verify is enabled by setting the environment variable
|
||||
// GODEBUG=gocacheverify=1.
|
||||
var verify = false
|
||||
|
||||
func init() { initEnv() }
|
||||
|
||||
func initEnv() {
|
||||
verify = false
|
||||
debugHash = false
|
||||
debug := strings.Split(os.Getenv("GODEBUG"), ",")
|
||||
for _, f := range debug {
|
||||
if f == "gocacheverify=1" {
|
||||
verify = true
|
||||
}
|
||||
if f == "gocachehash=1" {
|
||||
debugHash = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get looks up the action ID in the cache,
|
||||
// returning the corresponding output ID and file size, if any.
|
||||
// Note that finding an output ID does not guarantee that the
|
||||
// saved file for that output ID is still available.
|
||||
func (c *Cache) Get(id ActionID) (Entry, error) {
|
||||
if verify {
|
||||
return Entry{}, errMissing
|
||||
}
|
||||
return c.get(id)
|
||||
}
|
||||
|
||||
type Entry struct {
|
||||
OutputID OutputID
|
||||
Size int64
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
// get is Get but does not respect verify mode, so that Put can use it.
|
||||
func (c *Cache) get(id ActionID) (Entry, error) {
|
||||
missing := func() (Entry, error) {
|
||||
fmt.Fprintf(c.log, "%d miss %x\n", c.now().Unix(), id)
|
||||
return Entry{}, errMissing
|
||||
}
|
||||
f, err := os.Open(c.fileName(id, "a"))
|
||||
if err != nil {
|
||||
return missing()
|
||||
}
|
||||
defer f.Close()
|
||||
entry := make([]byte, entrySize+1) // +1 to detect whether f is too long
|
||||
if n, err := io.ReadFull(f, entry); n != entrySize || err != io.ErrUnexpectedEOF {
|
||||
return missing()
|
||||
}
|
||||
if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' {
|
||||
return missing()
|
||||
}
|
||||
eid, entry := entry[3:3+hexSize], entry[3+hexSize:]
|
||||
eout, entry := entry[1:1+hexSize], entry[1+hexSize:]
|
||||
esize, entry := entry[1:1+20], entry[1+20:]
|
||||
etime, entry := entry[1:1+20], entry[1+20:]
|
||||
var buf [HashSize]byte
|
||||
if _, err := hex.Decode(buf[:], eid); err != nil || buf != id {
|
||||
return missing()
|
||||
}
|
||||
if _, err := hex.Decode(buf[:], eout); err != nil {
|
||||
return missing()
|
||||
}
|
||||
i := 0
|
||||
for i < len(esize) && esize[i] == ' ' {
|
||||
i++
|
||||
}
|
||||
size, err := strconv.ParseInt(string(esize[i:]), 10, 64)
|
||||
if err != nil || size < 0 {
|
||||
return missing()
|
||||
}
|
||||
i = 0
|
||||
for i < len(etime) && etime[i] == ' ' {
|
||||
i++
|
||||
}
|
||||
tm, err := strconv.ParseInt(string(etime[i:]), 10, 64)
|
||||
if err != nil || size < 0 {
|
||||
return missing()
|
||||
}
|
||||
|
||||
fmt.Fprintf(c.log, "%d get %x\n", c.now().Unix(), id)
|
||||
|
||||
c.used(c.fileName(id, "a"))
|
||||
|
||||
return Entry{buf, size, time.Unix(0, tm)}, nil
|
||||
}
|
||||
|
||||
// GetBytes looks up the action ID in the cache and returns
|
||||
// the corresponding output bytes.
|
||||
// GetBytes should only be used for data that can be expected to fit in memory.
|
||||
func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) {
|
||||
entry, err := c.Get(id)
|
||||
if err != nil {
|
||||
return nil, entry, err
|
||||
}
|
||||
data, _ := ioutil.ReadFile(c.OutputFile(entry.OutputID))
|
||||
if sha256.Sum256(data) != entry.OutputID {
|
||||
return nil, entry, errMissing
|
||||
}
|
||||
return data, entry, nil
|
||||
}
|
||||
|
||||
// OutputFile returns the name of the cache file storing output with the given OutputID.
|
||||
func (c *Cache) OutputFile(out OutputID) string {
|
||||
file := c.fileName(out, "d")
|
||||
c.used(file)
|
||||
return file
|
||||
}
|
||||
|
||||
// Time constants for cache expiration.
|
||||
//
|
||||
// We set the mtime on a cache file on each use, but at most one per mtimeInterval (1 hour),
|
||||
// to avoid causing many unnecessary inode updates. The mtimes therefore
|
||||
// roughly reflect "time of last use" but may in fact be older by at most an hour.
|
||||
//
|
||||
// We scan the cache for entries to delete at most once per trimInterval (1 day).
|
||||
//
|
||||
// When we do scan the cache, we delete entries that have not been used for
|
||||
// at least trimLimit (5 days). Statistics gathered from a month of usage by
|
||||
// Go developers found that essentially all reuse of cached entries happened
|
||||
// within 5 days of the previous reuse. See golang.org/issue/22990.
|
||||
const (
|
||||
mtimeInterval = 1 * time.Hour
|
||||
trimInterval = 24 * time.Hour
|
||||
trimLimit = 5 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// used makes a best-effort attempt to update mtime on file,
|
||||
// so that mtime reflects cache access time.
|
||||
//
|
||||
// Because the reflection only needs to be approximate,
|
||||
// and to reduce the amount of disk activity caused by using
|
||||
// cache entries, used only updates the mtime if the current
|
||||
// mtime is more than an hour old. This heuristic eliminates
|
||||
// nearly all of the mtime updates that would otherwise happen,
|
||||
// while still keeping the mtimes useful for cache trimming.
|
||||
func (c *Cache) used(file string) {
|
||||
info, err := os.Stat(file)
|
||||
if err == nil && c.now().Sub(info.ModTime()) < mtimeInterval {
|
||||
return
|
||||
}
|
||||
os.Chtimes(file, c.now(), c.now())
|
||||
}
|
||||
|
||||
// Trim removes old cache entries that are likely not to be reused.
|
||||
func (c *Cache) Trim() {
|
||||
now := c.now()
|
||||
|
||||
// We maintain in dir/trim.txt the time of the last completed cache trim.
|
||||
// If the cache has been trimmed recently enough, do nothing.
|
||||
// This is the common case.
|
||||
data, _ := ioutil.ReadFile(filepath.Join(c.dir, "trim.txt"))
|
||||
t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
|
||||
if err == nil && now.Sub(time.Unix(t, 0)) < trimInterval {
|
||||
return
|
||||
}
|
||||
|
||||
// Trim each of the 256 subdirectories.
|
||||
// We subtract an additional mtimeInterval
|
||||
// to account for the imprecision of our "last used" mtimes.
|
||||
cutoff := now.Add(-trimLimit - mtimeInterval)
|
||||
for i := 0; i < 256; i++ {
|
||||
subdir := filepath.Join(c.dir, fmt.Sprintf("%02x", i))
|
||||
c.trimSubdir(subdir, cutoff)
|
||||
}
|
||||
|
||||
ioutil.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix())), 0666)
|
||||
}
|
||||
|
||||
// trimSubdir trims a single cache subdirectory.
|
||||
func (c *Cache) trimSubdir(subdir string, cutoff time.Time) {
|
||||
// Read all directory entries from subdir before removing
|
||||
// any files, in case removing files invalidates the file offset
|
||||
// in the directory scan. Also, ignore error from f.Readdirnames,
|
||||
// because we don't care about reporting the error and we still
|
||||
// want to process any entries found before the error.
|
||||
f, err := os.Open(subdir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
names, _ := f.Readdirnames(-1)
|
||||
f.Close()
|
||||
|
||||
for _, name := range names {
|
||||
// Remove only cache entries (xxxx-a and xxxx-d).
|
||||
if !strings.HasSuffix(name, "-a") && !strings.HasSuffix(name, "-d") {
|
||||
continue
|
||||
}
|
||||
entry := filepath.Join(subdir, name)
|
||||
info, err := os.Stat(entry)
|
||||
if err == nil && info.ModTime().Before(cutoff) {
|
||||
os.Remove(entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// putIndexEntry adds an entry to the cache recording that executing the action
|
||||
// with the given id produces an output with the given output id (hash) and size.
|
||||
func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify bool) error {
|
||||
// Note: We expect that for one reason or another it may happen
|
||||
// that repeating an action produces a different output hash
|
||||
// (for example, if the output contains a time stamp or temp dir name).
|
||||
// While not ideal, this is also not a correctness problem, so we
|
||||
// don't make a big deal about it. In particular, we leave the action
|
||||
// cache entries writable specifically so that they can be overwritten.
|
||||
//
|
||||
// Setting GODEBUG=gocacheverify=1 does make a big deal:
|
||||
// in verify mode we are double-checking that the cache entries
|
||||
// are entirely reproducible. As just noted, this may be unrealistic
|
||||
// in some cases but the check is also useful for shaking out real bugs.
|
||||
entry := []byte(fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano()))
|
||||
if verify && allowVerify {
|
||||
old, err := c.get(id)
|
||||
if err == nil && (old.OutputID != out || old.Size != size) {
|
||||
// panic to show stack trace, so we can see what code is generating this cache entry.
|
||||
msg := fmt.Sprintf("go: internal cache error: cache verify failed: id=%x changed:<<<\n%s\n>>>\nold: %x %d\nnew: %x %d", id, reverseHash(id), out, size, old.OutputID, old.Size)
|
||||
panic(msg)
|
||||
}
|
||||
}
|
||||
file := c.fileName(id, "a")
|
||||
if err := ioutil.WriteFile(file, entry, 0666); err != nil {
|
||||
os.Remove(file)
|
||||
return err
|
||||
}
|
||||
os.Chtimes(file, c.now(), c.now()) // mainly for tests
|
||||
|
||||
fmt.Fprintf(c.log, "%d put %x %x %d\n", c.now().Unix(), id, out, size)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Put stores the given output in the cache as the output for the action ID.
|
||||
// It may read file twice. The content of file must not change between the two passes.
|
||||
func (c *Cache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error) {
|
||||
return c.put(id, file, true)
|
||||
}
|
||||
|
||||
// PutNoVerify is like Put but disables the verify check
|
||||
// when GODEBUG=goverifycache=1 is set.
|
||||
// It is meant for data that is OK to cache but that we expect to vary slightly from run to run,
|
||||
// like test output containing times and the like.
|
||||
func (c *Cache) PutNoVerify(id ActionID, file io.ReadSeeker) (OutputID, int64, error) {
|
||||
return c.put(id, file, false)
|
||||
}
|
||||
|
||||
func (c *Cache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) {
|
||||
// Compute output ID.
|
||||
h := sha256.New()
|
||||
if _, err := file.Seek(0, 0); err != nil {
|
||||
return OutputID{}, 0, err
|
||||
}
|
||||
size, err := io.Copy(h, file)
|
||||
if err != nil {
|
||||
return OutputID{}, 0, err
|
||||
}
|
||||
var out OutputID
|
||||
h.Sum(out[:0])
|
||||
|
||||
// Copy to cached output file (if not already present).
|
||||
if err := c.copyFile(file, out, size); err != nil {
|
||||
return out, size, err
|
||||
}
|
||||
|
||||
// Add to cache index.
|
||||
return out, size, c.putIndexEntry(id, out, size, allowVerify)
|
||||
}
|
||||
|
||||
// PutBytes stores the given bytes in the cache as the output for the action ID.
|
||||
func (c *Cache) PutBytes(id ActionID, data []byte) error {
|
||||
_, _, err := c.Put(id, bytes.NewReader(data))
|
||||
return err
|
||||
}
|
||||
|
||||
// copyFile copies file into the cache, expecting it to have the given
|
||||
// output ID and size, if that file is not present already.
|
||||
func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error {
|
||||
name := c.fileName(out, "d")
|
||||
info, err := os.Stat(name)
|
||||
if err == nil && info.Size() == size {
|
||||
// Check hash.
|
||||
if f, err := os.Open(name); err == nil {
|
||||
h := sha256.New()
|
||||
io.Copy(h, f)
|
||||
f.Close()
|
||||
var out2 OutputID
|
||||
h.Sum(out2[:0])
|
||||
if out == out2 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// Hash did not match. Fall through and rewrite file.
|
||||
}
|
||||
|
||||
// Copy file to cache directory.
|
||||
mode := os.O_RDWR | os.O_CREATE
|
||||
if err == nil && info.Size() > size { // shouldn't happen but fix in case
|
||||
mode |= os.O_TRUNC
|
||||
}
|
||||
f, err := os.OpenFile(name, mode, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if size == 0 {
|
||||
// File now exists with correct size.
|
||||
// Only one possible zero-length file, so contents are OK too.
|
||||
// Early return here makes sure there's a "last byte" for code below.
|
||||
return nil
|
||||
}
|
||||
|
||||
// From here on, if any of the I/O writing the file fails,
|
||||
// we make a best-effort attempt to truncate the file f
|
||||
// before returning, to avoid leaving bad bytes in the file.
|
||||
|
||||
// Copy file to f, but also into h to double-check hash.
|
||||
if _, err := file.Seek(0, 0); err != nil {
|
||||
f.Truncate(0)
|
||||
return err
|
||||
}
|
||||
h := sha256.New()
|
||||
w := io.MultiWriter(f, h)
|
||||
if _, err := io.CopyN(w, file, size-1); err != nil {
|
||||
f.Truncate(0)
|
||||
return err
|
||||
}
|
||||
// Check last byte before writing it; writing it will make the size match
|
||||
// what other processes expect to find and might cause them to start
|
||||
// using the file.
|
||||
buf := make([]byte, 1)
|
||||
if _, err := file.Read(buf); err != nil {
|
||||
f.Truncate(0)
|
||||
return err
|
||||
}
|
||||
h.Write(buf)
|
||||
sum := h.Sum(nil)
|
||||
if !bytes.Equal(sum, out[:]) {
|
||||
f.Truncate(0)
|
||||
return fmt.Errorf("file content changed underfoot")
|
||||
}
|
||||
|
||||
// Commit cache file entry.
|
||||
if _, err := f.Write(buf); err != nil {
|
||||
f.Truncate(0)
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
// Data might not have been written,
|
||||
// but file may look like it is the right size.
|
||||
// To be extra careful, remove cached file.
|
||||
os.Remove(name)
|
||||
return err
|
||||
}
|
||||
os.Chtimes(name, c.now(), c.now()) // mainly for tests
|
||||
|
||||
return nil
|
||||
}
|
|
@ -0,0 +1,319 @@
|
|||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func init() {
|
||||
verify = false // even if GODEBUG is set
|
||||
}
|
||||
|
||||
func TestBasic(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "cachetest-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
_, err = Open(filepath.Join(dir, "notexist"))
|
||||
if err == nil {
|
||||
t.Fatal(`Open("tmp/notexist") succeeded, want failure`)
|
||||
}
|
||||
|
||||
cdir := filepath.Join(dir, "c1")
|
||||
if err := os.Mkdir(cdir, 0777); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c1, err := Open(cdir)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(c1) (create): %v", err)
|
||||
}
|
||||
if err := c1.putIndexEntry(dummyID(1), dummyID(12), 13, true); err != nil {
|
||||
t.Fatalf("addIndexEntry: %v", err)
|
||||
}
|
||||
if err := c1.putIndexEntry(dummyID(1), dummyID(2), 3, true); err != nil { // overwrite entry
|
||||
t.Fatalf("addIndexEntry: %v", err)
|
||||
}
|
||||
if entry, err := c1.Get(dummyID(1)); err != nil || entry.OutputID != dummyID(2) || entry.Size != 3 {
|
||||
t.Fatalf("c1.Get(1) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(2), 3)
|
||||
}
|
||||
|
||||
c2, err := Open(cdir)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(c2) (reuse): %v", err)
|
||||
}
|
||||
if entry, err := c2.Get(dummyID(1)); err != nil || entry.OutputID != dummyID(2) || entry.Size != 3 {
|
||||
t.Fatalf("c2.Get(1) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(2), 3)
|
||||
}
|
||||
if err := c2.putIndexEntry(dummyID(2), dummyID(3), 4, true); err != nil {
|
||||
t.Fatalf("addIndexEntry: %v", err)
|
||||
}
|
||||
if entry, err := c1.Get(dummyID(2)); err != nil || entry.OutputID != dummyID(3) || entry.Size != 4 {
|
||||
t.Fatalf("c1.Get(2) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(3), 4)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrowth(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "cachetest-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
c, err := Open(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
|
||||
n := 10000
|
||||
if testing.Short() {
|
||||
n = 1000
|
||||
}
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
if err := c.putIndexEntry(dummyID(i), dummyID(i*99), int64(i)*101, true); err != nil {
|
||||
t.Fatalf("addIndexEntry: %v", err)
|
||||
}
|
||||
id := ActionID(dummyID(i))
|
||||
entry, err := c.Get(id)
|
||||
if err != nil {
|
||||
t.Fatalf("Get(%x): %v", id, err)
|
||||
}
|
||||
if entry.OutputID != dummyID(i*99) || entry.Size != int64(i)*101 {
|
||||
t.Errorf("Get(%x) = %x, %d, want %x, %d", id, entry.OutputID, entry.Size, dummyID(i*99), int64(i)*101)
|
||||
}
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
id := ActionID(dummyID(i))
|
||||
entry, err := c.Get(id)
|
||||
if err != nil {
|
||||
t.Fatalf("Get2(%x): %v", id, err)
|
||||
}
|
||||
if entry.OutputID != dummyID(i*99) || entry.Size != int64(i)*101 {
|
||||
t.Errorf("Get2(%x) = %x, %d, want %x, %d", id, entry.OutputID, entry.Size, dummyID(i*99), int64(i)*101)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyPanic(t *testing.T) {
|
||||
os.Setenv("GODEBUG", "gocacheverify=1")
|
||||
initEnv()
|
||||
defer func() {
|
||||
os.Unsetenv("GODEBUG")
|
||||
verify = false
|
||||
}()
|
||||
|
||||
if !verify {
|
||||
t.Fatal("initEnv did not set verify")
|
||||
}
|
||||
|
||||
dir, err := ioutil.TempDir("", "cachetest-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
c, err := Open(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
|
||||
id := ActionID(dummyID(1))
|
||||
if err := c.PutBytes(id, []byte("abc")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
t.Log(err)
|
||||
return
|
||||
}
|
||||
}()
|
||||
c.PutBytes(id, []byte("def"))
|
||||
t.Fatal("mismatched Put did not panic in verify mode")
|
||||
}
|
||||
|
||||
func TestCacheLog(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "cachetest-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
c, err := Open(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
c.now = func() time.Time { return time.Unix(1e9, 0) }
|
||||
|
||||
id := ActionID(dummyID(1))
|
||||
c.Get(id)
|
||||
c.PutBytes(id, []byte("abc"))
|
||||
c.Get(id)
|
||||
|
||||
c, err = Open(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Open #2: %v", err)
|
||||
}
|
||||
c.now = func() time.Time { return time.Unix(1e9+1, 0) }
|
||||
c.Get(id)
|
||||
|
||||
id2 := ActionID(dummyID(2))
|
||||
c.Get(id2)
|
||||
c.PutBytes(id2, []byte("abc"))
|
||||
c.Get(id2)
|
||||
c.Get(id)
|
||||
|
||||
data, err := ioutil.ReadFile(filepath.Join(dir, "log.txt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := `1000000000 miss 0100000000000000000000000000000000000000000000000000000000000000
|
||||
1000000000 put 0100000000000000000000000000000000000000000000000000000000000000 ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3
|
||||
1000000000 get 0100000000000000000000000000000000000000000000000000000000000000
|
||||
1000000001 get 0100000000000000000000000000000000000000000000000000000000000000
|
||||
1000000001 miss 0200000000000000000000000000000000000000000000000000000000000000
|
||||
1000000001 put 0200000000000000000000000000000000000000000000000000000000000000 ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3
|
||||
1000000001 get 0200000000000000000000000000000000000000000000000000000000000000
|
||||
1000000001 get 0100000000000000000000000000000000000000000000000000000000000000
|
||||
`
|
||||
if string(data) != want {
|
||||
t.Fatalf("log:\n%s\nwant:\n%s", string(data), want)
|
||||
}
|
||||
}
|
||||
|
||||
func dummyID(x int) [HashSize]byte {
|
||||
var out [HashSize]byte
|
||||
binary.LittleEndian.PutUint64(out[:], uint64(x))
|
||||
return out
|
||||
}
|
||||
|
||||
func TestCacheTrim(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "cachetest-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
c, err := Open(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
const start = 1000000000
|
||||
now := int64(start)
|
||||
c.now = func() time.Time { return time.Unix(now, 0) }
|
||||
|
||||
checkTime := func(name string, mtime int64) {
|
||||
t.Helper()
|
||||
file := filepath.Join(c.dir, name[:2], name)
|
||||
info, err := os.Stat(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.ModTime().Unix() != mtime {
|
||||
t.Fatalf("%s mtime = %d, want %d", name, info.ModTime().Unix(), mtime)
|
||||
}
|
||||
}
|
||||
|
||||
id := ActionID(dummyID(1))
|
||||
c.PutBytes(id, []byte("abc"))
|
||||
entry, _ := c.Get(id)
|
||||
c.PutBytes(ActionID(dummyID(2)), []byte("def"))
|
||||
mtime := now
|
||||
checkTime(fmt.Sprintf("%x-a", id), mtime)
|
||||
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime)
|
||||
|
||||
// Get should not change recent mtimes.
|
||||
now = start + 10
|
||||
c.Get(id)
|
||||
checkTime(fmt.Sprintf("%x-a", id), mtime)
|
||||
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime)
|
||||
|
||||
// Get should change distant mtimes.
|
||||
now = start + 5000
|
||||
mtime2 := now
|
||||
if _, err := c.Get(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.OutputFile(entry.OutputID)
|
||||
checkTime(fmt.Sprintf("%x-a", id), mtime2)
|
||||
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime2)
|
||||
|
||||
// Trim should leave everything alone: it's all too new.
|
||||
c.Trim()
|
||||
if _, err := c.Get(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.OutputFile(entry.OutputID)
|
||||
data, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkTime(fmt.Sprintf("%x-a", dummyID(2)), start)
|
||||
|
||||
// Trim less than a day later should not do any work at all.
|
||||
now = start + 80000
|
||||
c.Trim()
|
||||
if _, err := c.Get(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.OutputFile(entry.OutputID)
|
||||
data2, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(data, data2) {
|
||||
t.Fatalf("second trim did work: %q -> %q", data, data2)
|
||||
}
|
||||
|
||||
// Fast forward and do another trim just before the 5 day cutoff.
|
||||
// Note that because of usedQuantum the cutoff is actually 5 days + 1 hour.
|
||||
// We used c.Get(id) just now, so 5 days later it should still be kept.
|
||||
// On the other hand almost a full day has gone by since we wrote dummyID(2)
|
||||
// and we haven't looked at it since, so 5 days later it should be gone.
|
||||
now += 5 * 86400
|
||||
checkTime(fmt.Sprintf("%x-a", dummyID(2)), start)
|
||||
c.Trim()
|
||||
if _, err := c.Get(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.OutputFile(entry.OutputID)
|
||||
mtime3 := now
|
||||
if _, err := c.Get(dummyID(2)); err == nil { // haven't done a Get for this since original write above
|
||||
t.Fatalf("Trim did not remove dummyID(2)")
|
||||
}
|
||||
|
||||
// The c.Get(id) refreshed id's mtime again.
|
||||
// Check that another 5 days later it is still not gone,
|
||||
// but check by using checkTime, which doesn't bring mtime forward.
|
||||
now += 5 * 86400
|
||||
c.Trim()
|
||||
checkTime(fmt.Sprintf("%x-a", id), mtime3)
|
||||
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3)
|
||||
|
||||
// Half a day later Trim should still be a no-op, because there was a Trim recently.
|
||||
// Even though the entry for id is now old enough to be trimmed,
|
||||
// it gets a reprieve until the time comes for a new Trim scan.
|
||||
now += 86400 / 2
|
||||
c.Trim()
|
||||
checkTime(fmt.Sprintf("%x-a", id), mtime3)
|
||||
checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3)
|
||||
|
||||
// Another half a day later, Trim should actually run, and it should remove id.
|
||||
now += 86400/2 + 1
|
||||
c.Trim()
|
||||
if _, err := c.Get(dummyID(1)); err == nil {
|
||||
t.Fatal("Trim did not remove dummyID(1)")
|
||||
}
|
||||
}
|
|
@ -0,0 +1,100 @@
|
|||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"cmd/go/internal/base"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Default returns the default cache to use, or nil if no cache should be used.
|
||||
func Default() *Cache {
|
||||
defaultOnce.Do(initDefaultCache)
|
||||
return defaultCache
|
||||
}
|
||||
|
||||
var (
|
||||
defaultOnce sync.Once
|
||||
defaultCache *Cache
|
||||
)
|
||||
|
||||
// cacheREADME is a message stored in a README in the cache directory.
|
||||
// Because the cache lives outside the normal Go trees, we leave the
|
||||
// README as a courtesy to explain where it came from.
|
||||
const cacheREADME = `This directory holds cached build artifacts from the Go build system.
|
||||
Run "go clean -cache" if the directory is getting too large.
|
||||
See golang.org to learn more about Go.
|
||||
`
|
||||
|
||||
// initDefaultCache does the work of finding the default cache
|
||||
// the first time Default is called.
|
||||
func initDefaultCache() {
|
||||
dir := DefaultDir()
|
||||
if dir == "off" {
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0777); err != nil {
|
||||
base.Fatalf("initializing cache in $GOCACHE: %s", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "README")); err != nil {
|
||||
// Best effort.
|
||||
ioutil.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666)
|
||||
}
|
||||
|
||||
c, err := Open(dir)
|
||||
if err != nil {
|
||||
base.Fatalf("initializing cache in $GOCACHE: %s", err)
|
||||
}
|
||||
defaultCache = c
|
||||
}
|
||||
|
||||
// DefaultDir returns the effective GOCACHE setting.
|
||||
// It returns "off" if the cache is disabled.
|
||||
func DefaultDir() string {
|
||||
dir := os.Getenv("GOCACHE")
|
||||
if dir != "" {
|
||||
return dir
|
||||
}
|
||||
|
||||
// Compute default location.
|
||||
// TODO(rsc): This code belongs somewhere else,
|
||||
// like maybe ioutil.CacheDir or os.CacheDir.
|
||||
switch runtime.GOOS {
|
||||
case "windows":
|
||||
dir = os.Getenv("LocalAppData")
|
||||
|
||||
case "darwin":
|
||||
dir = os.Getenv("HOME")
|
||||
if dir == "" {
|
||||
return "off"
|
||||
}
|
||||
dir += "/Library/Caches"
|
||||
|
||||
case "plan9":
|
||||
dir = os.Getenv("home")
|
||||
if dir == "" {
|
||||
return "off"
|
||||
}
|
||||
// Plan 9 has no established per-user cache directory,
|
||||
// but $home/lib/xyz is the usual equivalent of $HOME/.xyz on Unix.
|
||||
dir += "/lib/cache"
|
||||
|
||||
default: // Unix
|
||||
// https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
|
||||
dir = os.Getenv("XDG_CACHE_HOME")
|
||||
if dir == "" {
|
||||
dir = os.Getenv("HOME")
|
||||
if dir == "" {
|
||||
return "off"
|
||||
}
|
||||
dir += "/.cache"
|
||||
}
|
||||
}
|
||||
return filepath.Join(dir, "go-build")
|
||||
}
|
|
@ -0,0 +1,174 @@
|
|||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var debugHash = false // set when GODEBUG=gocachehash=1
|
||||
|
||||
// HashSize is the number of bytes in a hash.
|
||||
const HashSize = 32
|
||||
|
||||
// A Hash provides access to the canonical hash function used to index the cache.
|
||||
// The current implementation uses salted SHA256, but clients must not assume this.
|
||||
type Hash struct {
|
||||
h hash.Hash
|
||||
name string // for debugging
|
||||
buf *bytes.Buffer // for verify
|
||||
}
|
||||
|
||||
// hashSalt is a salt string added to the beginning of every hash
|
||||
// created by NewHash. Using the Go version makes sure that different
|
||||
// versions of the go command (or even different Git commits during
|
||||
// work on the development branch) do not address the same cache
|
||||
// entries, so that a bug in one version does not affect the execution
|
||||
// of other versions. This salt will result in additional ActionID files
|
||||
// in the cache, but not additional copies of the large output files,
|
||||
// which are still addressed by unsalted SHA256.
|
||||
var hashSalt = []byte(runtime.Version())
|
||||
|
||||
// Subkey returns an action ID corresponding to mixing a parent
|
||||
// action ID with a string description of the subkey.
|
||||
func Subkey(parent ActionID, desc string) ActionID {
|
||||
h := sha256.New()
|
||||
h.Write([]byte("subkey:"))
|
||||
h.Write(parent[:])
|
||||
h.Write([]byte(desc))
|
||||
var out ActionID
|
||||
h.Sum(out[:0])
|
||||
if debugHash {
|
||||
fmt.Fprintf(os.Stderr, "HASH subkey %x %q = %x\n", parent, desc, out)
|
||||
}
|
||||
if verify {
|
||||
hashDebug.Lock()
|
||||
hashDebug.m[out] = fmt.Sprintf("subkey %x %q", parent, desc)
|
||||
hashDebug.Unlock()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NewHash returns a new Hash.
|
||||
// The caller is expected to Write data to it and then call Sum.
|
||||
func NewHash(name string) *Hash {
|
||||
h := &Hash{h: sha256.New(), name: name}
|
||||
if debugHash {
|
||||
fmt.Fprintf(os.Stderr, "HASH[%s]\n", h.name)
|
||||
}
|
||||
h.Write(hashSalt)
|
||||
if verify {
|
||||
h.buf = new(bytes.Buffer)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Write writes data to the running hash.
|
||||
func (h *Hash) Write(b []byte) (int, error) {
|
||||
if debugHash {
|
||||
fmt.Fprintf(os.Stderr, "HASH[%s]: %q\n", h.name, b)
|
||||
}
|
||||
if h.buf != nil {
|
||||
h.buf.Write(b)
|
||||
}
|
||||
return h.h.Write(b)
|
||||
}
|
||||
|
||||
// Sum returns the hash of the data written previously.
|
||||
func (h *Hash) Sum() [HashSize]byte {
|
||||
var out [HashSize]byte
|
||||
h.h.Sum(out[:0])
|
||||
if debugHash {
|
||||
fmt.Fprintf(os.Stderr, "HASH[%s]: %x\n", h.name, out)
|
||||
}
|
||||
if h.buf != nil {
|
||||
hashDebug.Lock()
|
||||
if hashDebug.m == nil {
|
||||
hashDebug.m = make(map[[HashSize]byte]string)
|
||||
}
|
||||
hashDebug.m[out] = h.buf.String()
|
||||
hashDebug.Unlock()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// In GODEBUG=gocacheverify=1 mode,
|
||||
// hashDebug holds the input to every computed hash ID,
|
||||
// so that we can work backward from the ID involved in a
|
||||
// cache entry mismatch to a description of what should be there.
|
||||
var hashDebug struct {
|
||||
sync.Mutex
|
||||
m map[[HashSize]byte]string
|
||||
}
|
||||
|
||||
// reverseHash returns the input used to compute the hash id.
|
||||
func reverseHash(id [HashSize]byte) string {
|
||||
hashDebug.Lock()
|
||||
s := hashDebug.m[id]
|
||||
hashDebug.Unlock()
|
||||
return s
|
||||
}
|
||||
|
||||
var hashFileCache struct {
|
||||
sync.Mutex
|
||||
m map[string][HashSize]byte
|
||||
}
|
||||
|
||||
// HashFile returns the hash of the named file.
|
||||
// It caches repeated lookups for a given file,
|
||||
// and the cache entry for a file can be initialized
|
||||
// using SetFileHash.
|
||||
// The hash used by FileHash is not the same as
|
||||
// the hash used by NewHash.
|
||||
func FileHash(file string) ([HashSize]byte, error) {
|
||||
hashFileCache.Lock()
|
||||
out, ok := hashFileCache.m[file]
|
||||
hashFileCache.Unlock()
|
||||
|
||||
if ok {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
if debugHash {
|
||||
fmt.Fprintf(os.Stderr, "HASH %s: %v\n", file, err)
|
||||
}
|
||||
return [HashSize]byte{}, err
|
||||
}
|
||||
_, err = io.Copy(h, f)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
if debugHash {
|
||||
fmt.Fprintf(os.Stderr, "HASH %s: %v\n", file, err)
|
||||
}
|
||||
return [HashSize]byte{}, err
|
||||
}
|
||||
h.Sum(out[:0])
|
||||
if debugHash {
|
||||
fmt.Fprintf(os.Stderr, "HASH %s: %x\n", file, out)
|
||||
}
|
||||
|
||||
SetFileHash(file, out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SetFileHash sets the hash returned by FileHash for file.
|
||||
func SetFileHash(file string, sum [HashSize]byte) {
|
||||
hashFileCache.Lock()
|
||||
if hashFileCache.m == nil {
|
||||
hashFileCache.m = make(map[string][HashSize]byte)
|
||||
}
|
||||
hashFileCache.m[file] = sum
|
||||
hashFileCache.Unlock()
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue