Code:
/ Net / Net / 3.5.50727.3053 / DEVDIV / depot / DevDiv / releases / Orcas / SP / wpf / src / Shared / MS / Internal / TokenizerHelper.cs / 1 / TokenizerHelper.cs
//---------------------------------------------------------------------------- // // Copyright (C) Microsoft Corporation. All rights reserved. // // File: TokenizerHelper.cs // // Description: This file contains the implementation of TokenizerHelper. // This class should be used by most - if not all - MIL parsers. // // History: // 05/19/2003 : adsmith - Created it // 05/20/2003 : adsmith - Moved to Shared // //--------------------------------------------------------------------------- using System; using System.Collections; using System.Diagnostics; using System.ComponentModel; using System.Globalization; #if !PBTCOMPILER using System.Windows; #endif #if WINDOWS_BASE using MS.Internal.WindowsBase; #elif PRESENTATION_CORE using MS.Internal.PresentationCore; #elif PRESENTATIONFRAMEWORK using MS.Internal.PresentationFramework; #elif PBTCOMPILER using MS.Utility ; using MS.Internal.Markup; #elif DRT using MS.Internal.Drt; #else #error Attempt to use FriendAccessAllowedAttribute from an unknown assembly. using MS.Internal.YourAssemblyName; #endif #if PBTCOMPILER namespace MS.Internal.Markup #else namespace MS.Internal #endif { #if !PBTCOMPILER [FriendAccessAllowed] #endif internal class TokenizerHelper { ////// Constructor for TokenizerHelper which accepts an IFormatProvider. /// If the IFormatProvider is null, we use the thread's IFormatProvider info. /// We will use ',' as the list separator, unless it's the same as the /// decimal separator. If it *is*, then we can't determine if, say, "23,5" is one /// number or two. In this case, we will use ";" as the separator. /// /// The string which will be tokenized. /// The IFormatProvider which controls this tokenization. internal TokenizerHelper(string str, IFormatProvider formatProvider) { char numberSeparator = GetNumericListSeparator(formatProvider); Initialize(str, '\'', numberSeparator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. internal TokenizerHelper(string str, char quoteChar, char separator) { Initialize(str, quoteChar, separator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. private void Initialize(string str, char quoteChar, char separator) { _str = str; _strLen = str == null ? 0 : str.Length; _currentTokenIndex = -1; _quoteChar = quoteChar; _argSeparator = separator; // immediately forward past any whitespace so // NextToken() logic always starts on the first // character of the next token. while (_charIndex < _strLen) { if (!Char.IsWhiteSpace(_str, _charIndex)) { break; } ++_charIndex; } } internal string GetCurrentToken() { // if no current token, return null if (_currentTokenIndex < 0) { return null; } return _str.Substring(_currentTokenIndex,_currentTokenLength); } ////// Throws an exception if there is any non-whitespace left un-parsed. /// internal void LastTokenRequired() { if (_charIndex != _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken() { return NextToken(false); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired() { if (!NextToken(false)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired(bool allowQuotedToken) { if (!NextToken(allowQuotedToken)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken) { // use the currently-set separator character. return NextToken(allowQuotedToken, _argSeparator); } ////// Advances to the NextToken. A separator character can be specified /// which overrides the one previously set. /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken, char separator) { _currentTokenIndex = -1; // reset the currentTokenIndex _foundSeparator = false; // reset // If we're at end of the string, just return false. if (_charIndex >= _strLen) { return false; } char currentChar = _str[_charIndex]; Debug.Assert(!Char.IsWhiteSpace(currentChar),"Token started on Whitespace"); // setup the quoteCount int quoteCount = 0; // If we are allowing a quoted token and this token begins with a quote, // set up the quote count and skip the initial quote if (allowQuotedToken && currentChar == _quoteChar) { quoteCount++; // increment quote count ++_charIndex; // move to next character } int newTokenIndex = _charIndex; int newTokenLength = 0; // loop until hit end of string or hit a , or whitespace // if at end of string ust return false. while (_charIndex < _strLen) { currentChar = _str[_charIndex]; // if have a QuoteCount and this is a quote // decrement the quoteCount if (quoteCount > 0) { // if anything but a quoteChar we move on if (currentChar == _quoteChar) { --quoteCount; // if at zero which it always should for now // break out of the loop if (0 == quoteCount) { ++_charIndex; // move past the quote break; } } } else if ((Char.IsWhiteSpace(currentChar)) || (currentChar == separator)) { if (currentChar == separator) { _foundSeparator = true; } break; } ++_charIndex; ++newTokenLength; } // if quoteCount isn't zero we hit the end of the string // before the ending quote if (quoteCount > 0) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperMissingEndQuote)); } ScanToNextToken(separator); // move so at the start of the nextToken for next call // finally made it, update the _currentToken values _currentTokenIndex = newTokenIndex; _currentTokenLength = newTokenLength; if (_currentTokenLength < 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } return true; } // helper to move the _charIndex to the next token or to the end of the string void ScanToNextToken(char separator) { // if already at end of the string don't bother if (_charIndex < _strLen) { char currentChar = _str[_charIndex]; // check that the currentChar is a space or the separator. If not // we have an error. this can happen in the quote case // that the char after the quotes string isn't a char. if (!(currentChar == separator) && !Char.IsWhiteSpace(currentChar)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } // loop until hit a character that isn't // an argument separator or whitespace. // !!!Todo: if more than one argSet throw an exception int argSepCount = 0; while (_charIndex < _strLen) { currentChar = _str[_charIndex]; if (currentChar == separator) { _foundSeparator = true; ++argSepCount; _charIndex++; if (argSepCount > 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } else if (Char.IsWhiteSpace(currentChar)) { ++_charIndex; } else { break; } } // if there was a separatorChar then we shouldn't be // at the end of string or means there was a separator // but there isn't an arg if (argSepCount > 0 && _charIndex >= _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } } // Helper to get the numeric list separator for a given IFormatProvider. // Separator is a comma [,] if the decimal separator is not a comma, or a semicolon [;] otherwise. static internal char GetNumericListSeparator(IFormatProvider provider) { char numericSeparator = ','; // Get the NumberFormatInfo out of the provider, if possible // If the IFormatProvider doesn't not contain a NumberFormatInfo, then // this method returns the current culture's NumberFormatInfo. NumberFormatInfo numberFormat = NumberFormatInfo.GetInstance(provider); Debug.Assert(null != numberFormat); // Is the decimal separator is the same as the list separator? // If so, we use the ";". if ((numberFormat.NumberDecimalSeparator.Length > 0) && (numericSeparator == numberFormat.NumberDecimalSeparator[0])) { numericSeparator = ';'; } return numericSeparator; } internal bool FoundSeparator { get { return _foundSeparator; } } char _quoteChar; char _argSeparator; string _str; int _strLen; int _charIndex; internal int _currentTokenIndex; internal int _currentTokenLength; bool _foundSeparator; } } // File provided for Reference Use Only by Microsoft Corporation (c) 2007. // Copyright (c) Microsoft Corporation. All rights reserved. //---------------------------------------------------------------------------- // // Copyright (C) Microsoft Corporation. All rights reserved. // // File: TokenizerHelper.cs // // Description: This file contains the implementation of TokenizerHelper. // This class should be used by most - if not all - MIL parsers. // // History: // 05/19/2003 : adsmith - Created it // 05/20/2003 : adsmith - Moved to Shared // //--------------------------------------------------------------------------- using System; using System.Collections; using System.Diagnostics; using System.ComponentModel; using System.Globalization; #if !PBTCOMPILER using System.Windows; #endif #if WINDOWS_BASE using MS.Internal.WindowsBase; #elif PRESENTATION_CORE using MS.Internal.PresentationCore; #elif PRESENTATIONFRAMEWORK using MS.Internal.PresentationFramework; #elif PBTCOMPILER using MS.Utility ; using MS.Internal.Markup; #elif DRT using MS.Internal.Drt; #else #error Attempt to use FriendAccessAllowedAttribute from an unknown assembly. using MS.Internal.YourAssemblyName; #endif #if PBTCOMPILER namespace MS.Internal.Markup #else namespace MS.Internal #endif { #if !PBTCOMPILER [FriendAccessAllowed] #endif internal class TokenizerHelper { ////// Constructor for TokenizerHelper which accepts an IFormatProvider. /// If the IFormatProvider is null, we use the thread's IFormatProvider info. /// We will use ',' as the list separator, unless it's the same as the /// decimal separator. If it *is*, then we can't determine if, say, "23,5" is one /// number or two. In this case, we will use ";" as the separator. /// /// The string which will be tokenized. /// The IFormatProvider which controls this tokenization. internal TokenizerHelper(string str, IFormatProvider formatProvider) { char numberSeparator = GetNumericListSeparator(formatProvider); Initialize(str, '\'', numberSeparator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. internal TokenizerHelper(string str, char quoteChar, char separator) { Initialize(str, quoteChar, separator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. private void Initialize(string str, char quoteChar, char separator) { _str = str; _strLen = str == null ? 0 : str.Length; _currentTokenIndex = -1; _quoteChar = quoteChar; _argSeparator = separator; // immediately forward past any whitespace so // NextToken() logic always starts on the first // character of the next token. while (_charIndex < _strLen) { if (!Char.IsWhiteSpace(_str, _charIndex)) { break; } ++_charIndex; } } internal string GetCurrentToken() { // if no current token, return null if (_currentTokenIndex < 0) { return null; } return _str.Substring(_currentTokenIndex,_currentTokenLength); } ////// Throws an exception if there is any non-whitespace left un-parsed. /// internal void LastTokenRequired() { if (_charIndex != _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken() { return NextToken(false); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired() { if (!NextToken(false)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired(bool allowQuotedToken) { if (!NextToken(allowQuotedToken)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken) { // use the currently-set separator character. return NextToken(allowQuotedToken, _argSeparator); } ////// Advances to the NextToken. A separator character can be specified /// which overrides the one previously set. /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken, char separator) { _currentTokenIndex = -1; // reset the currentTokenIndex _foundSeparator = false; // reset // If we're at end of the string, just return false. if (_charIndex >= _strLen) { return false; } char currentChar = _str[_charIndex]; Debug.Assert(!Char.IsWhiteSpace(currentChar),"Token started on Whitespace"); // setup the quoteCount int quoteCount = 0; // If we are allowing a quoted token and this token begins with a quote, // set up the quote count and skip the initial quote if (allowQuotedToken && currentChar == _quoteChar) { quoteCount++; // increment quote count ++_charIndex; // move to next character } int newTokenIndex = _charIndex; int newTokenLength = 0; // loop until hit end of string or hit a , or whitespace // if at end of string ust return false. while (_charIndex < _strLen) { currentChar = _str[_charIndex]; // if have a QuoteCount and this is a quote // decrement the quoteCount if (quoteCount > 0) { // if anything but a quoteChar we move on if (currentChar == _quoteChar) { --quoteCount; // if at zero which it always should for now // break out of the loop if (0 == quoteCount) { ++_charIndex; // move past the quote break; } } } else if ((Char.IsWhiteSpace(currentChar)) || (currentChar == separator)) { if (currentChar == separator) { _foundSeparator = true; } break; } ++_charIndex; ++newTokenLength; } // if quoteCount isn't zero we hit the end of the string // before the ending quote if (quoteCount > 0) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperMissingEndQuote)); } ScanToNextToken(separator); // move so at the start of the nextToken for next call // finally made it, update the _currentToken values _currentTokenIndex = newTokenIndex; _currentTokenLength = newTokenLength; if (_currentTokenLength < 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } return true; } // helper to move the _charIndex to the next token or to the end of the string void ScanToNextToken(char separator) { // if already at end of the string don't bother if (_charIndex < _strLen) { char currentChar = _str[_charIndex]; // check that the currentChar is a space or the separator. If not // we have an error. this can happen in the quote case // that the char after the quotes string isn't a char. if (!(currentChar == separator) && !Char.IsWhiteSpace(currentChar)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } // loop until hit a character that isn't // an argument separator or whitespace. // !!!Todo: if more than one argSet throw an exception int argSepCount = 0; while (_charIndex < _strLen) { currentChar = _str[_charIndex]; if (currentChar == separator) { _foundSeparator = true; ++argSepCount; _charIndex++; if (argSepCount > 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } else if (Char.IsWhiteSpace(currentChar)) { ++_charIndex; } else { break; } } // if there was a separatorChar then we shouldn't be // at the end of string or means there was a separator // but there isn't an arg if (argSepCount > 0 && _charIndex >= _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } } // Helper to get the numeric list separator for a given IFormatProvider. // Separator is a comma [,] if the decimal separator is not a comma, or a semicolon [;] otherwise. static internal char GetNumericListSeparator(IFormatProvider provider) { char numericSeparator = ','; // Get the NumberFormatInfo out of the provider, if possible // If the IFormatProvider doesn't not contain a NumberFormatInfo, then // this method returns the current culture's NumberFormatInfo. NumberFormatInfo numberFormat = NumberFormatInfo.GetInstance(provider); Debug.Assert(null != numberFormat); // Is the decimal separator is the same as the list separator? // If so, we use the ";". if ((numberFormat.NumberDecimalSeparator.Length > 0) && (numericSeparator == numberFormat.NumberDecimalSeparator[0])) { numericSeparator = ';'; } return numericSeparator; } internal bool FoundSeparator { get { return _foundSeparator; } } char _quoteChar; char _argSeparator; string _str; int _strLen; int _charIndex; internal int _currentTokenIndex; internal int _currentTokenLength; bool _foundSeparator; } } // File provided for Reference Use Only by Microsoft Corporation (c) 2007. // Copyright (c) Microsoft Corporation. All rights reserved.
Link Menu
This book is available now!
Buy at Amazon US or
Buy at Amazon UK
- SchemaTypeEmitter.cs
- HijriCalendar.cs
- LineBreakRecord.cs
- WindowsUpDown.cs
- _NtlmClient.cs
- RtfNavigator.cs
- XmlAttributes.cs
- SymLanguageVendor.cs
- BatchStream.cs
- ToolStripContainerDesigner.cs
- DaylightTime.cs
- EmbeddedMailObjectsCollection.cs
- SynchronizingStream.cs
- DefaultEventAttribute.cs
- ADMembershipUser.cs
- SiblingIterators.cs
- Ipv6Element.cs
- FactoryId.cs
- ThrowHelper.cs
- DataTransferEventArgs.cs
- Hashtable.cs
- ReadOnlyDataSource.cs
- CorrelationManager.cs
- WindowInteractionStateTracker.cs
- DispatcherTimer.cs
- ZipIOCentralDirectoryDigitalSignature.cs
- CallSiteOps.cs
- StrokeNodeOperations2.cs
- FilterQuery.cs
- DataGridViewRowsRemovedEventArgs.cs
- StateDesigner.Layouts.cs
- UserPreferenceChangedEventArgs.cs
- AudioDeviceOut.cs
- HTMLTextWriter.cs
- SHA384.cs
- XmlSerializerSection.cs
- RuntimeVariablesExpression.cs
- UnsafeMethods.cs
- dtdvalidator.cs
- CompilerResults.cs
- HMACSHA1.cs
- SmtpNtlmAuthenticationModule.cs
- CFStream.cs
- SingleTagSectionHandler.cs
- HttpApplicationFactory.cs
- StringKeyFrameCollection.cs
- StyleCollectionEditor.cs
- SafeNativeMethods.cs
- ArrayMergeHelper.cs
- PenLineJoinValidation.cs
- IntSumAggregationOperator.cs
- AnimatedTypeHelpers.cs
- TextTreeUndo.cs
- FloaterParagraph.cs
- BinaryObjectInfo.cs
- CryptoStream.cs
- SqlColumnizer.cs
- ClientSideProviderDescription.cs
- CharacterHit.cs
- CurrencyManager.cs
- CurrencyWrapper.cs
- CFStream.cs
- Update.cs
- WindowsComboBox.cs
- SerializerWriterEventHandlers.cs
- OverflowException.cs
- FocusChangedEventArgs.cs
- DirectoryInfo.cs
- TableDetailsRow.cs
- ProcessModuleCollection.cs
- SchemaManager.cs
- PagesSection.cs
- ShapingEngine.cs
- InputMethod.cs
- DataGridViewSelectedCellCollection.cs
- DrawItemEvent.cs
- BitmapCodecInfo.cs
- SafeNativeMethods.cs
- ImageCodecInfo.cs
- uribuilder.cs
- HttpApplication.cs
- ChangeConflicts.cs
- OperationInfoBase.cs
- Scheduler.cs
- PeerCollaboration.cs
- FontUnit.cs
- DataTableReaderListener.cs
- XmlQualifiedName.cs
- BasicKeyConstraint.cs
- ParentControlDesigner.cs
- InOutArgumentConverter.cs
- ResolveNameEventArgs.cs
- XPathAncestorQuery.cs
- DependencyPropertyConverter.cs
- InheritanceContextHelper.cs
- Cursor.cs
- EntityModelBuildProvider.cs
- SqlClientPermission.cs
- ControllableStoryboardAction.cs
- ParserHooks.cs