eztools
/
v6
/
http.go

package eztools

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"encoding/xml"
	"io"
	"mime"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/mongodb-forks/digest"
	"golang.org/x/net/html"
)

const (
	// HTMLHead head recommented by W3C
	HTMLHead = `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">`
	// AuthNone no auth in cfg
	AuthNone = iota
	// AuthPlain token
	AuthPlain
	// AuthBasic plain text
	AuthBasic
	// AuthDigest digest
	AuthDigest
	// BodyTypeJSON json
	BodyTypeJSON = "json"
	// MimeTypeJSON full mime type for BodyTypeJSON
	MimeTypeJSON = "application/" + BodyTypeJSON
	// BodyTypeFile file
	BodyTypeFile = "file"
	// MimeTypeFile full mime type for BodyTypeFile
	MimeTypeFile = BodyTypeFile
	// BodyTypeText text
	BodyTypeText = "text"
	// MimeTypeText full mime type for BodyTypeText
	MimeTypeText = BodyTypeText
	// BodyTypeXML xml
	BodyTypeXML = "xml"
	// MimeTypeXML full mime type for BodyTypeXML
	MimeTypeXML = "application/" + BodyTypeXML
	// BodyTypeHTML html
	BodyTypeHTML = "html"
	// MimeTypeHTML full mime type for BodyTypeHTML
	MimeTypeHTML = "text/" + BodyTypeHTML
	// BodyTypeForm form
	BodyTypeForm = "form"
	// MimeTypeForm mime type for BodyTypeForm
	MimeTypeForm = "application/x-www-form-urlencoded"
)

// AuthInsecureTLS insecure TLS enabled
var AuthInsecureTLS bool

// AuthInfo authorization info
type AuthInfo struct {
	Type       int
	User, Pass string
}

// FormatMimeType generates MIME type with default params
// charset=utf-8
func FormatMimeType(bodyType string) string {
	switch bodyType {
	case BodyTypeFile:
		bodyType = MimeTypeFile
	case BodyTypeHTML:
		bodyType = MimeTypeHTML
	case BodyTypeXML:
		bodyType = MimeTypeXML
	case BodyTypeText:
		bodyType = MimeTypeText
	case BodyTypeJSON:
		bodyType = MimeTypeJSON
	case BodyTypeForm:
		bodyType = MimeTypeForm
	}
	return mime.FormatMediaType(bodyType, map[string]string{
		"charset": "utf-8",
	})
}

func genHTTPFile(defRdr io.Reader, fType, fName string) (string, io.Reader) {
	if len(fName) < 1 {
		if len(fType) < 1 {
			fType = BodyTypeJSON
		}
		return FormatMimeType(fType), defRdr
	}
	if len(fType) < 1 {
		fType = BodyTypeFile
	}
	var err error
	defer func() {
		if err != nil {
			if Debugging && Verbose > 1 {
				LogPrint(err)
			}
		}
	}()
	// New multipart writer.
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	fw, err := writer.CreateFormFile(fType, filepath.Base(fName))
	if err != nil {
		return fType, defRdr
	}
	file, err := os.Open(fName)
	if err != nil {
		return fType, defRdr
	}
	_, err = io.Copy(fw, file)
	if err != nil {
		return fType, defRdr
	}
	writer.Close()
	return writer.FormDataContentType(), bytes.NewReader(body.Bytes())
}

func genHTTPReq(method, url string, bodyReq io.Reader, fType, fName string,
	hdrs map[string]string) (req *http.Request, err error) {
	bodyType, bodyReq := genHTTPFile(bodyReq, fType, fName)
	req, err = http.NewRequest(method, url, bodyReq)
	if err != nil {
		if Debugging {
			ShowStrln("failed to create " + method)
		}
		return
	}
	addHdr := func(nm, vl string) {
		_, ok := hdrs[nm]
		if ok {
			return
		}
		req.Header.Add(nm, vl)
	}
	if bodyReq != nil {
		addHdr("Content-Type", bodyType)
		/*if Debugging && Verbose > 2 {
			ShowStrln("body type=" + bodyType)
		}*/
	}
	addHdr("Accept", "*/*")
	for n, v := range hdrs {
		req.Header.Add(n, v)
		/*if Debugging && Verbose > 2 {
			ShowStrln("adding header:" + n + "=" + v)
		}*/
	}
	return
}

// HTTPGetBody get body from response, stripping magic.
// Return values: bodyType and statusCode are returned as long as resp is provided.
//
//	bodyBytes is nil if Content-Length is 0 in header.
//	ErrInvalidInput=no response input
//	ErrOutOfBound=magic not matched
//	ErrNoValidResults=bad status code (non 2xx)
//	other errors from io.ReadAll()
func HTTPGetBody(resp *http.Response, magic []byte) (bodyType string,
	bodyBytes []byte, statusCode int, err error) {
	if resp == nil {
		err = ErrInvalidInput
		return
	}
	defer resp.Body.Close()

	bodyType = resp.Header.Get("Content-Type")
	statusCode = resp.StatusCode
	bodyBytes, err = io.ReadAll(resp.Body)
	if statusCode < http.StatusOK || statusCode >= http.StatusBadRequest {
		if Debugging && Verbose > 1 {
			ShowStrln("failure response " + strconv.Itoa(statusCode))
		}
		err = ErrNoValidResults
		/*var b []byte
		if resp.ContentLength > 0 {
			b = make([]byte, resp.ContentLength)
			resp.Body.Read(b)
		}
		return "", b, statusCode, errors.New(resp.Status)*/
		return
	}
	if err != nil { // body not read
		//LogErrPrint(err)
		return
	}

	/*if Debugging && Verbose > 2 {
		ShowStrln("resp code=" + strconv.Itoa(statusCode))
	}*/
	if cl := resp.Header.Get("Content-Length"); cl == "0" {
		/*if Debugging && Verbose > 2 {
			ShowStrln("no body in response")
		}*/
		return
	}
	if bodyBytes == nil || len(bodyBytes) < 1 {
		/*if Debugging && Verbose > 2 {
			LogPrintWtTime("no body")
		}*/
		return
	}
	if len(magic) > 0 {
		/*if Debugging && Verbose > 1 {
			ShowStrln("stripping magic")
		}*/
		if bytes.HasPrefix(bodyBytes, magic) {
			bodyBytes = bytes.TrimLeft(bytes.TrimPrefix(bodyBytes,
				magic), "\n\r")
		} else {
			err = ErrOutOfBound
			return
		}
	}
	/*if Debugging && Verbose > 2 {
		LogPrintWtTime("type", bodyType, "body", bodyBytes)
	}*/
	return
}

// ParseMimeType parses MIME type to BodyType, neglecting params
func ParseMimeType(v string) string {
	mimeType, _, err := mime.ParseMediaType(v)
	if err != nil {
		return v
	}
	switch mimeType {
	case MimeTypeFile:
		return BodyTypeFile
	case MimeTypeHTML:
		return BodyTypeHTML
	case MimeTypeJSON:
		return BodyTypeJSON
	case MimeTypeText:
		return BodyTypeText
	case MimeTypeXML:
		return BodyTypeXML
	}
	return mimeType
}

// HTTPParseBody parses body from response to json or file.
// text body is not processed.
//
//	Parameter:
//
// strucOut: func(n *html.Node) for BodyTypeHTML
//
//	Return values: ErrIncomplete=not parsed because type not recognized
//
// other errors are from HTTPGetBody(), json.Unmarshal() or FileWrite()
func HTTPParseBody(resp *http.Response, fileName string, strucOut interface{},
	magic []byte) (recognized, bodyType string, bodyBytes []byte, statusCode int, err error) {
	bodyType, bodyBytes, statusCode, err = HTTPGetBody(resp, magic)
	if err != nil || (bodyBytes == nil || len(bodyBytes) < 1) {
		return
	}
	recognized = ParseMimeType(bodyType)
	writeFile := func() {
		if len(fileName) > 0 {
			err = FileWrite(fileName, bodyBytes)
			/*var out *os.File
			out, err = os.Create(fileName)
			if err != nil {
				break
			}
			defer out.Close()
			_, err = io.Copy(out, bytes.NewReader(bodyBytes))*/
		}
	}
	switch recognized {
	case BodyTypeJSON:
		if strucOut != nil {
			err = json.Unmarshal(bodyBytes, strucOut)
		}
	case BodyTypeXML:
		if strucOut != nil {
			xml.Unmarshal(bodyBytes, strucOut)
		}
	case BodyTypeHTML:
		if strucOut != nil {
			fun, ok := strucOut.(func(n *html.Node))
			if ok {
				doc, err := html.Parse(bytes.NewReader(bodyBytes))
				if err == nil {
					fun(doc)
				}
			}
		}
	case BodyTypeText:
	case BodyTypeFile:
		writeFile()
	default:
		switch {
		case strings.Contains(recognized, BodyTypeText),
			strings.Contains(recognized, "application/xhtml+xml"),
			strings.Contains(recognized, "application/javascript"),
			strings.Contains(recognized, "text/javascript"):
			recognized = BodyTypeText
			// bytes are maybe better than string, so leave them as are
			/*if Debugging && Verbose > 2 {
				LogPrintWtTime("type", recognized, "body", bodyBytes)
			}*/
		case strings.Contains(recognized, BodyTypeFile),
			strings.Contains(recognized, "audio"),
			strings.Contains(recognized, "video"),
			strings.Contains(recognized, "application"),
			strings.Contains(recognized, "image"):
			recognized = BodyTypeFile
			writeFile()
		default:
			err = ErrIncomplete
		}
	}
	return
}

func httpSend(method, url string, to time.Duration,
	bodyReq io.Reader, fType, fName string, hdrs map[string]string,
	funcReqProc func(req *http.Request),
	funcReqSend func(req *http.Request) (*http.Response,
		error)) (resp *http.Response, err error) {
	/*if Debugging && Verbose > 1 {
		ShowStrln(method + " " + url)
	}*/
	req, err := genHTTPReq(method, url, bodyReq, fType, fName, hdrs)
	if err != nil {
		return
	}
	if funcReqProc != nil {
		funcReqProc(req)
	}
	if Debugging && Verbose > 2 {
		LogWtTime(*req)
	}
	//ShowSthln(req.ContentLength)
	if funcReqSend == nil {
		tc := &tls.Config{InsecureSkipVerify: AuthInsecureTLS}
		tr := &http.Transport{TLSClientConfig: tc}
		cli := &http.Client{Timeout: to, Transport: tr}
		funcReqSend = cli.Do
	}
	resp, err = funcReqSend(req)
	if err != nil {
		if Debugging {
			ShowStrln("failed to send/get")
		}
		/*} else {
		if Debugging && Verbose > 2 {
			LogWtTime(*resp)
		}*/
	}
	return
}

const defHTTPGetTO = 60 * time.Second

func httpSendNParseResp(method, url string, authInfo AuthInfo,
	bodyReq io.Reader, fType, fileOut string,
	hdrs map[string]string) (resp *http.Response, err error) {
	switch authInfo.Type {
	case AuthDigest:
		resp, err = httpSend(method, url, 0, bodyReq,
			fType, fileOut, hdrs, nil,
			func(req *http.Request) (*http.Response, error) {
				t := digest.NewTransport(authInfo.User, authInfo.Pass)
				return t.RoundTrip(req)
			})
	case AuthBasic:
		resp, err = httpSend(method, url, defHTTPGetTO, bodyReq,
			fType, fileOut, hdrs,
			func(req *http.Request) {
				if len(authInfo.Pass) > 0 {
					req.SetBasicAuth(authInfo.User, authInfo.Pass)
				}
			}, nil)
	case AuthPlain:
		resp, err = httpSend(method, url, defHTTPGetTO, bodyReq,
			fType, fileOut, hdrs,
			func(req *http.Request) {
				if len(authInfo.Pass) > 0 {
					req.Header.Set("authorization", "Basic "+authInfo.Pass)
				}
			}, nil)
	default: // AUTH_NONE
		resp, err = httpSend(method, url, defHTTPGetTO, bodyReq,
			fType, fileOut, hdrs, nil, nil)
	}
	//statusCode = resp.StatusCode
	return
}

// HTTPSendAuthNHdrNFile sends a request and returns the result.
// Specify a file with name and type to be sent as body, and/or extra headers.
// If something wrong with the file, a request will be sent without it anyway.
func HTTPSendAuthNHdrNFile(method, url string, authInfo AuthInfo,
	fType, fName string, hdrs map[string]string) (*http.Response, error) {
	return httpSendNParseResp(method,
		url, authInfo, nil, fType, fName, hdrs)
}

// HTTPSendAuthNHdr sends a request and returns the result.
// With body and/or extra headers.
func HTTPSendAuthNHdr(method, url, bodyType string, authInfo AuthInfo,
	bodyReq io.Reader, hdrs map[string]string) (*http.Response, error) {
	return httpSendNParseResp(method,
		url, authInfo, bodyReq, bodyType, "", hdrs)
}

// HTTPSendHdr sends a request and returns the result.
// With body and/or extra headers.
func HTTPSendHdr(method, url, bodyType string,
	bodyReq io.Reader, hdrs map[string]string) (*http.Response, error) {
	return httpSendNParseResp(method,
		url, AuthInfo{}, bodyReq, bodyType, "", hdrs)
}

// HTTPSendAuth sends a request and returns the result.
// = HttpSend + AuthInfo
func HTTPSendAuth(method, url, bodyType string, authInfo AuthInfo,
	bodyReq io.Reader) (*http.Response, error) {
	return httpSendNParseResp(method,
		url, authInfo, bodyReq, bodyType, "", nil)
}

// HTTPSend sends HTTP request and returns the result.
// It does not need AuthInfo as HTTPSend.
// Set AUTH_INSECURE_TLS to true to skip TLS (X509) verification
func HTTPSend(method, url, bodyType string, bodyReq io.Reader) (*http.Response, error) {
	return httpSendNParseResp(method, url,
		AuthInfo{}, bodyReq, bodyType, "", nil)
}

// RangeStrMap iterate through map[string]interface{} obj, calling fun for
// each element recursively. When fun returns true, it stops.
// false is returned if no element found.
func RangeStrMap(obj interface{}, fun func(k string, v interface{}) bool) bool {
	//if the argument is not a map, ignore it
	mobj, ok := obj.(map[string]interface{})
	if !ok {
		return false
	}

	for k, v := range mobj {
		//key match, return value
		if fun(k, v) {
			return true
		}

		//if the value is a map, search recursively
		if m, ok := v.(map[string]interface{}); ok {
			if RangeStrMap(m, fun) {
				return true
			}
		}
		//if the value is an array, search recursively
		//from each element
		if va, ok := v.([]interface{}); ok {
			for _, a := range va {
				if RangeStrMap(a, fun) {
					return true
				}
			}
		}
	}

	//element not found
	return false
}

// FindStrMap find string key in map[string]interface{} obj,
// returning the value and true or nil and false.
func FindStrMap(obj interface{}, key string) (interface{}, bool) {
	//if the argument is not a map, ignore it
	mobj, ok := obj.(map[string]interface{})
	if !ok {
		return nil, false
	}

	for k, v := range mobj {
		//key match, return value
		if k == key {
			return v, true
		}

		//if the value is a map, search recursively
		if m, ok := v.(map[string]interface{}); ok {
			if res, ok := FindStrMap(m, key); ok {
				return res, true
			}
		}
		//if the value is an array, search recursively
		//from each element
		if va, ok := v.([]interface{}); ok {
			for _, a := range va {
				if res, ok := FindStrMap(a, key); ok {
					return res, true
				}
			}
		}
	}

	//element not found
	return nil, false
}