Author: ceki Date: Fri Mar 6 16:04:37 2009 New Revision: 2184 Added: logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/CorpusModel.java - copied, changed from r2183, /logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/CorpusMaker.java logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/MessageArgumentTuple.java - copied, changed from r2183, /logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/MessageItem.java logback/trunk/logback-classic/src/test/resources/corpus/ - copied from r2179, /logback/trunk/logback-classic/src/test/input/corpus/ Removed: logback/trunk/logback-classic/src/test/input/corpus/ logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/CorpusMaker.java logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/MessageItem.java Modified: logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/Corpus.java logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/LogStatement.java logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/TextFileUtil.java Log: Finishing up work on the Corpus. Modified: logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/Corpus.java ============================================================================== --- logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/Corpus.java (original) +++ logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/Corpus.java Fri Mar 6 16:04:37 2009 @@ -1,18 +1,55 @@ package ch.qos.logback.classic.corpus; +import java.io.FileWriter; import java.io.IOException; +import java.net.URL; import java.util.List; import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.classic.spi.LoggerContextVO; import ch.qos.logback.classic.spi.PubLoggingEventVO; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; +import ch.qos.logback.core.CoreConstants; +/** + * + * <p>Usage: + * + * <p><code>ILoggingEvent[] eventArray = Corpus.makeStandardCorpus();</code> + * + * <p>if you wish to dump the events into a file, say "/corpus.log" : + * + * <p> + * <code>Corpus.dump(eventArray, "/corpus.log"); + * + * <p>For the model behind the corpus, refer to {@link CorpusModel}. + * + * @author Ceki Gülcü + * + */ public class Corpus { - - static final int STANDARD_CORPUS_SIZE = 50 * 1000; - static final int STANDARD_SEED = 1123; - static public ILoggingEvent[] make(CorpusMaker corpusMaker, int n) { + static public final int STANDARD_CORPUS_SIZE = 50 * 1000; + private static final int STANDARD_SEED = 34780; + + /** + * Make a standard corpus. The standard corpus has + * {@link #STANDARD_CORPUS_SIZE} elements. + * + * @return event array representing the standard corpus + * @throws IOException + */ + static public ILoggingEvent[] makeStandardCorpus() throws IOException { + ClassLoader classLoader = Corpus.class.getClassLoader(); + URL originOfSpeciesURL = classLoader + .getResource("corpus/origin_of_species.txt"); + List<String> worldList = TextFileUtil.toWords(originOfSpeciesURL); + CorpusModel corpusMaker = new CorpusModel(STANDARD_SEED, worldList); + return make(corpusMaker, STANDARD_CORPUS_SIZE); + } + + static public ILoggingEvent[] make(CorpusModel corpusMaker, int n) { LoggerContextVO lcVO = corpusMaker.getRandomlyNamedLoggerContextVO(); PubLoggingEventVO[] plevoArray = new PubLoggingEventVO[n]; for (int i = 0; i < n; i++) { @@ -20,23 +57,37 @@ plevoArray[i] = e; e.loggerContextVO = lcVO; e.timeStamp = corpusMaker.getRandomTimeStamp(); - + LogStatement logStatement = corpusMaker.getRandomLogStatementFromPool(); e.loggerName = logStatement.loggerName; e.level = logStatement.level; - e.message = logStatement.messagerItem.message; - e.argumentArray = corpusMaker.getRandomArgumentArray(logStatement.messagerItem.numberOfArguments); + e.message = logStatement.mat.message; + e.argumentArray = corpusMaker + .getRandomArgumentArray(logStatement.mat.numberOfArguments); e.throwableProxy = logStatement.throwableProxy; e.threadName = corpusMaker.getRandomThreadNameFromPool(); } return plevoArray; } - static public ILoggingEvent[] makeStandardCorpus() throws IOException { - List<String> worldList = TextFileUtil - .toWords("src/test/input/corpus/origin_of_species.txt"); - CorpusMaker corpusMaker = new CorpusMaker(STANDARD_SEED, worldList); - return make(corpusMaker, STANDARD_CORPUS_SIZE); + /** + * Dump the events passed as argument into the file named targetFile. + * + * @param eventArray + * @param targetFile + * @throws IOException + */ + public void dump(ILoggingEvent[] eventArray, String targetFile) + throws IOException { + FileWriter fw = new FileWriter(targetFile); + for (ILoggingEvent e : eventArray) { + fw.write(e.toString()); + fw.append(CoreConstants.LINE_SEPARATOR); + if (e.getThrowableProxy() != null) { + IThrowableProxy tp = e.getThrowableProxy(); + fw.write(ThrowableProxyUtil.asString(tp)); + } + } } } Copied: logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/CorpusModel.java (from r2183, /logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/CorpusMaker.java) ============================================================================== --- /logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/CorpusMaker.java (original) +++ logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/CorpusModel.java Fri Mar 6 16:04:37 2009 @@ -20,44 +20,75 @@ import ch.qos.logback.classic.spi.ThrowableProxy; import ch.qos.logback.classic.spi.ThrowableProxyVO; -public class CorpusMaker { +/** + * Models the corpus. + * + * <p>This contains the probability distributions of levels, logger names, + * messages, message arguments. + * + * @author Ceki Gülcü + * + */ +public class CorpusModel { + + // N(u,s) denotes a random variable normally distributed with mean u and + // variance sqrt(s), where sqrt() is the square root function. For an + // explanation of normal distribution please see + // http://en.wikipedia.org/wiki/Normal_distribution + + // It is assumed that the number of parts in a logger name is a random + // variable normally distributed with mean AVERAGE_LOGGER_NAME_PARTS and + // standard deviation STD_DEV_FOR_LOGGER_NAME_PARTS + static final int AVERAGE_LOGGER_NAME_PARTS = 6; + static final int STD_DEV_FOR_LOGGER_NAME_PARTS = 3; + + // It is assumed that there are LOGGER_POOL_SIZE logger names + // in our model corpus. + static final int LOGGER_POOL_SIZE = 1000; + + // It is assumed that there are LOG_STATEMENT_POOL_SIZE log statements + // in our model corpus. + static final int LOG_STATEMENT_POOL_SIZE = LOGGER_POOL_SIZE * 8; // level distribution is determined by the following table - // it corresponds to TRACE 20%, DEBUG 30%, INFO 30%, WARN 10%, + // It corresponds to TRACE 20%, DEBUG 30%, INFO 30%, WARN 10%, // ERROR 10%. See also getRandomLevel() method. static final double[] LEVEL_DISTRIBUTION = new double[] { .2, .5, .8, .9 }; + // It is assumed that the number of words in the message (contained in a log + // statement) is a random variable normally distributed with mean + // AVERAGE_MESSAGE_WORDS and standard deviation STD_DEV_FOR_MESSAGE_WORDS + static final int AVERAGE_MESSAGE_WORDS = 8; + static final int STD_DEV_FOR_MESSAGE_WORDS = 4; + // messages will have no arguments 80% of the time, one argument in 8%, two // arguments in 7% and three arguments in 5% of cases static final double[] ARGUMENT_DISTRIBUTION = new double[] { .80, .88, 0.95 }; static final double THROWABLE_PROPABILITY_FOR_WARNING = .1; static final double THROWABLE_PROPABILITY_FOR_ERRORS = .3; + // .5 of throwables are nested once static final double NESTING_PROBABILITY = .5; - static final int AVERAGE_LOGGER_NAME_PARTS = 6; - static final int STD_DEV_FOR_LOGGER_NAME_PARTS = 3; - - static final int AVERAGE_MESSAGE_WORDS = 8; - static final int STD_DEV_FOR_MESSAGE_WORDS = 4; - + // For each logging event the timer is incremented by a certain value. it is + // assumed that this value is a random variable normally distributed with mean + // AVERAGE_MILLIS_INCREMENT and standard deviation + // STD_DEV_FOR_MILLIS_INCREMENT static final int AVERAGE_MILLIS_INCREMENT = 10; static final int STD_DEV_FOR_MILLIS_INCREMENT = 5; + // assume that there are THREAD_POOL_SIZE threads in the corpus static final int THREAD_POOL_SIZE = 10; - static final int LOGGER_POOL_SIZE = 1000; - static final int LOG_STATEMENT_POOL_SIZE = LOGGER_POOL_SIZE * 8; final Random random; - List<String> worldList; + final List<String> worldList; String[] threadNamePool; - LogStatement[] logStatementPool; // 2009-03-06 13:08 GMT long lastTimeStamp = 1236344888578L; - public CorpusMaker(long seed, List<String> worldList) { + public CorpusModel(long seed, List<String> worldList) { random = new Random(seed); this.worldList = worldList; buildThreadNamePool(); @@ -149,7 +180,7 @@ return argumentArray; } - private MessageItem makeRandomMessageEntry() { + private MessageArgumentTuple makeRandomMessageArgumentTuple() { int numOfArguments = getNumberOfMessageArguments(); int wordCount = RandomUtil.gaussianAsPositiveInt(random, @@ -167,11 +198,11 @@ sb.append(wordArray[i]).append(' '); } sb.append(getRandomWord()); - return new MessageItem(sb.toString(), numOfArguments); + return new MessageArgumentTuple(sb.toString(), numOfArguments); } private LogStatement makeRandomLogStatement(String[] loggerNamePool) { - MessageItem mi = makeRandomMessageEntry(); + MessageArgumentTuple mat = makeRandomMessageArgumentTuple(); String loggerName = getRandomLoggerNameFromPool(loggerNamePool); Level randomLevel = getRandomLevel(); Throwable t = getRandomThrowable(randomLevel); @@ -180,7 +211,7 @@ throwableProxy = ThrowableProxyVO.build(new ThrowableProxy(t)); pupulateWithPackagingData(throwableProxy.getStackTraceElementProxyArray()); } - LogStatement logStatement = new LogStatement(loggerName, randomLevel, mi, + LogStatement logStatement = new LogStatement(loggerName, randomLevel, mat, throwableProxy); return logStatement; } @@ -208,7 +239,7 @@ step.setClassPackagingData(cpd); } } - + private int getNumberOfMessageArguments() { double rn = random.nextDouble(); if (rn < ARGUMENT_DISTRIBUTION[0]) { Modified: logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/LogStatement.java ============================================================================== --- logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/LogStatement.java (original) +++ logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/LogStatement.java Fri Mar 6 16:04:37 2009 @@ -6,24 +6,16 @@ public class LogStatement { final String loggerName; - final MessageItem messagerItem; + final MessageArgumentTuple mat; final Level level; final IThrowableProxy throwableProxy; - public LogStatement(String loggerName, Level level, MessageItem messagerItem, + public LogStatement(String loggerName, Level level, MessageArgumentTuple mat, IThrowableProxy tp) { this.loggerName = loggerName; this.level = level; - this.messagerItem = messagerItem; + this.mat = mat; this.throwableProxy = tp; } - public String getLoggerName() { - return loggerName; - } - - public MessageItem getMessagerItem() { - return messagerItem; - } - } Copied: logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/MessageArgumentTuple.java (from r2183, /logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/MessageItem.java) ============================================================================== --- /logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/MessageItem.java (original) +++ logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/MessageArgumentTuple.java Fri Mar 6 16:04:37 2009 @@ -1,16 +1,15 @@ package ch.qos.logback.classic.corpus; -public class MessageItem { - +public class MessageArgumentTuple { final String message; final int numberOfArguments; - MessageItem(String message) { + MessageArgumentTuple(String message) { this(message, 0); } - public MessageItem(String message, int numberOfArguments) { + public MessageArgumentTuple(String message, int numberOfArguments) { this.message = message; this.numberOfArguments = numberOfArguments; } Modified: logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/TextFileUtil.java ============================================================================== --- logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/TextFileUtil.java (original) +++ logback/trunk/logback-classic/src/test/java/ch/qos/logback/classic/corpus/TextFileUtil.java Fri Mar 6 16:04:37 2009 @@ -3,11 +3,17 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; +import java.net.URL; import java.util.ArrayList; import java.util.List; public class TextFileUtil { + + public static List<String> toWords(URL url) throws IOException { + String filename = url.getFile(); + return toWords(filename); + } public static List<String> toWords(String filename) throws IOException { FileReader fr = new FileReader(filename);