/**************************************************************************
 *
 * mgstat.c -- Program to generate statistics on a text collection
 * Copyright (C) 1994  Neil Sharman
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 * $Id: mgstat.c 25147 2012-02-28 00:59:00Z kjdon $
 *
 **************************************************************************/

#include "sysfuncs.h"

#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */

#include "mg_files.h"
#include "locallib.h"
#include "mg.h"
#include "words.h"
#include "invf.h"
#include "text.h"
#include "longlong.h"

/*
   $Log$
   Revision 1.1  2003/02/20 21:18:24  mdewsnip
   Addition of MG package for search and retrieval

   Revision 1.2  2001/09/21 12:46:42  kjm18
   updated mg to be in line with mg_1.3f. Now uses long long for some variables
   to enable indexing of very large collections.

   Revision 1.1  1999/08/10 21:18:19  sjboddie
   renamed mg-1.3d directory mg

   Revision 1.2  1998/11/25 07:55:50  rjmcnab

   Modified mg to that you can specify the stemmer you want
   to use via a command line option. You specify it to
   mg_passes during the build process. The number of the
   stemmer that you used is stored within the inverted
   dictionary header and the stemmed dictionary header so
   the correct stemmer is used in later stages of building
   and querying.

   Revision 1.1  1998/11/17 09:35:31  rjmcnab
   *** empty log message ***

   * Revision 1.2  1994/09/20  04:41:59  tes
   * For version 1.1
   *
 */

static char *RCSID = "$Id: mgstat.c 25147 2012-02-28 00:59:00Z kjdon $";

char *search_for_collection (char *name);
int process_file (char *, char *, int);
void ProcessDict (char *);
void ProcessStem (char *);
mg_s_long ProcessStemBlk (char *name);  /* [RPAP - Jan 97: Stem Index Change] */

static mg_ullong inputbytes = 0;
static mg_ullong total = 0;





int main (int argc, char **argv)
{
  mg_ullong sub_total;
  int fast;
  char *file_name = "";
  int ch;
  int exact = 0;
  mg_s_long indexed = 0; /* [RPAP - Jan 97: Stem Index Change] */
  opterr = 0;
  while ((ch = getopt (argc, argv, "Ehf:d:")) != -1)
    switch (ch)
      {
      case 'E':
	exact = 1;
	break;
      case 'f':		/* input file */
	file_name = optarg;
	break;
      case 'd':
	set_basepath (optarg);
	break;
      case 'h':
      case '?':
	fprintf (stderr, "usage: %s [-f input_file]"
		 "[-d data directory] [-h]\n", argv[0]);
	exit (1);
      }

  if (optind < argc)
    file_name = search_for_collection (argv[optind]);

  ProcessDict (file_name);
  ProcessStem (file_name);
  indexed = ProcessStemBlk (file_name); /* [RPAP - Jan 97: Stem Index Change] */

  {
    char Name[256];
    char *s;
    sprintf (Name, FILE_NAME_FORMAT, get_basepath (), file_name, "");
    s = strrchr (Name, '/');
    if (s)
      *s = '\0';
    printf ("\nThe collection is in \"%s\"\n", Name);
  }


  printf ("\n\t\tFiles required by mgquery\n");
  process_file (file_name, TEXT_SUFFIX, exact);
  process_file (file_name, INVF_SUFFIX, exact);
  process_file (file_name, TEXT_IDX_WGT_SUFFIX, exact);
  process_file (file_name, APPROX_WEIGHTS_SUFFIX, exact);
  process_file (file_name, INVF_DICT_BLOCKED_SUFFIX, exact);

  /* [RPAP - Jan 97: Stem Index Change] */
  if (indexed)
    {
      process_file (file_name, INVF_DICT_BLOCKED_1_SUFFIX, exact);
      process_file (file_name, INVF_DICT_BLOCKED_2_SUFFIX, exact);
      process_file (file_name, INVF_DICT_BLOCKED_3_SUFFIX, exact);
    }

  fast = process_file (file_name, TEXT_DICT_FAST_SUFFIX, exact);
  if (!fast)
    {
      process_file (file_name, TEXT_DICT_SUFFIX, exact);
      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
    }


  process_file (NULL, "SUB TOTAL", exact);
  sub_total = total;
  total = 0;

  printf ("\n\t\tFiles NOT required by mgquery\n");
  if (fast)
    {
      process_file (file_name, TEXT_DICT_SUFFIX, exact);
      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
    }
  process_file (file_name, INVF_DICT_SUFFIX, exact);
  process_file (file_name, INVF_IDX_SUFFIX, exact);
  process_file (file_name, TEXT_STATS_DICT_SUFFIX, exact);
  process_file (file_name, TEXT_IDX_SUFFIX, exact);
  process_file (file_name, WEIGHTS_SUFFIX, exact);
  process_file (file_name, INVF_CHUNK_SUFFIX, exact);
  process_file (file_name, INVF_CHUNK_TRANS_SUFFIX, exact);
  process_file (file_name, INVF_DICT_HASH_SUFFIX, exact);
  process_file (file_name, INVF_PARAGRAPH_SUFFIX, exact);
  process_file (NULL, "SUB TOTAL", exact);
  total += sub_total;
  printf ("\n");
  process_file (NULL, "TOTAL", exact);
  return 0;

}



char *
search_for_collection (char *name)
{
  char *dir = get_basepath ();
  static char buffer[512];
  struct stat stat_buf;

  /* Look in the current directory first */
  if (stat (name, &stat_buf) != -1)
    {
      if (S_ISDIR(stat_buf.st_mode))
	{
	  /* The name is a directory */
	  sprintf (buffer, "%s/%s", name, name);
	  set_basepath (".");
	  return buffer;
	}
    }

  sprintf (buffer, "%s.text", name);
  if (stat (buffer, &stat_buf) != -1)
    {
      if (S_ISREG(stat_buf.st_mode))
	{
	  /* The name is a directory */
	  set_basepath (".");
	  return name;
	}
    }
  sprintf (buffer, "%s/%s", dir, name);
  if (stat (buffer, &stat_buf) != -1)
    {
      if (S_ISDIR(stat_buf.st_mode))
	{
	  /* The name is a directory */
	  sprintf (buffer, "%s/%s", name, name);
	  return buffer;
	}
    }
  return name;
}






void 
ProcessDict (char *name)
{
  FILE *f;
  compression_dict_header cdh;
  int have_cdh = 0;
  compressed_text_header cth;
  int have_cth = 0;

  if ((f = open_file (name, TEXT_DICT_SUFFIX, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
    {
      Read_cdh (f, &cdh, NULL, NULL);
      fclose (f);
      have_cdh = 1;
    }

  if ((f = open_file (name, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
    {
      fread ((char *) &cth, sizeof (cth), 1, f);

      /* [RPAP - Jan 97: Endian Ordering] */
      NTOHUL(cth.num_of_docs);
      NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
      NTOHUL(cth.num_of_words);
      NTOHUL(cth.length_of_longest_doc);
      NTOHD(cth.ratio);

      fclose (f);
      have_cth = 1;
    }

  if (have_cth)
    {
      inputbytes = cth.num_of_bytes;
      printf ("Input bytes                        : %10f, %8.2f Mbyte\n",
	      cth.num_of_bytes, (double) cth.num_of_bytes / 1024 / 1024);
      printf ("Documents                          : %10d\n", cth.num_of_docs);
      printf ("Words in collection [dict]         : %10d\n", cth.num_of_words);
      printf ("Longest doc in collection [dict]   : %10d characters\n",
	      cth.length_of_longest_doc);
      printf ("Maximum ratio                      : %10.2f\n", cth.ratio);
    }

  if (have_cdh)
    {
      printf ("Words in dict                      : %10d\n", cdh.num_words[1]);
      printf ("Non-words in dict                  : %10d\n", cdh.num_words[0]);
      printf ("Total chars of distinct words      : %10d\n", cdh.num_word_chars[1]);
      printf ("Total chars of distinct non-words  : %10d\n", cdh.num_word_chars[0]);
    }

}








void 
ProcessStem (char *name)
{
  FILE *f;
  struct invf_dict_header idh;

  if (!(f = open_file (name, INVF_DICT_SUFFIX, "rb",
		       MAGIC_STEM_BUILD, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
    return;
  fread ((char *) &idh, sizeof (idh), 1, f);
  printf ("Words in collection [stem]         : %10d\n", NTOHUL(idh.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
  printf ("Words in stem                      : %10d\n", NTOHUL(idh.dict_size));  /* [RPAP - Jan 97: Endian Ordering] */
  printf ("Indexed fragments                  : %10d\n", NTOHUL(idh.num_of_docs));  /* [RPAP - Jan 97: Endian Ordering] */
  printf ("Total chars of stem words          : %10d\n", NTOHUL(idh.total_bytes));  /* [RPAP - Jan 97: Endian Ordering] */
  fclose (f);
}





/* [RPAP - Jan 97: Stem Index Change] */
mg_s_long 
ProcessStemBlk (char *name)
{
  FILE *f;
  struct stem_dict_header sdh;

  if (!(f = open_file (name, INVF_DICT_BLOCKED_SUFFIX, "rb",
		       MAGIC_STEM, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
    return 0;
  fread ((char *) &sdh, sizeof (sdh), 1, f);
  fclose (f);
  return NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
}




int 
process_file (char *name, char *ext, int exact)
{
  static double scale = 0;
  static char *units;
  struct stat buf;
  if (scale == 0)
    {
      /* So can get output in Mb or Kb, */
      /* will divide by a scale of 1024(Kb) or 1024*1024(Mb) */
      /* Note: if inputbytes==0, then use 1024 as default */
      scale = inputbytes > 10 * 1000 * 1000 ? 1024 * 1024 : 1024;
      units = scale == 1024 ? "Kb" : "Mb";
    }
  if (name)
    {
      char Name[256];
      sprintf (Name, FILE_NAME_FORMAT, get_basepath (), name, ext);
      if (!stat (Name, &buf))
	{
	  char fname[256];
	  char *nam = strrchr (name, '/');
	  nam = nam ? nam + 1 : name;
	  sprintf (fname, "%s%s", nam, ext);

	  if (inputbytes == 0.0)
	    {
	      if (exact)
		printf ("%s%*s: %10ld bytes   %7.3f%%\n", fname,
			35 - (int) strlen (fname), "",
			buf.st_size,
			100.0 * buf.st_size / inputbytes);
	      else
		printf ("%s%*s: %8.2f %s   %7.3f%%\n", fname,
			35 - (int) strlen (fname), "",
			buf.st_size / scale,
			units,
			100.0 * buf.st_size / inputbytes);
	    }
	  else
	    {
	      if (exact)
		printf ("%s%*s: %10ld bytes\n", fname,
			35 - (int) strlen (fname), "",
			buf.st_size);
	      else
		printf ("%s%*s: %8.2f %s\n", fname,
			35 - (int) strlen (fname), "",
			buf.st_size / scale,
			units);
	    }
	  total += buf.st_size;
	  return 1;
	}
      else
	return 0;
    }
  else
    {
      if (inputbytes == 0.0)
	{
	  if (exact)
	    printf ("%-34s : %10" ULL_FS " bytes   %7.3f%%\n", ext,
		    total,
		    100.0 * total / inputbytes);
	  else
	    printf ("%-34s : %8.2f %s   %7.3f%%\n", ext,
		    total / scale, units,
		    100.0 * total / inputbytes);
	}
      else
	{
	  if (exact)
	    printf ("%-34s : %10" ULL_FS " bytes\n", ext, total);
	  else
	    printf ("%-34s : %8.2f %s\n", ext,
		    total / scale, units);
	}
    }
  return 1;

}
